Merge branch 'worktree-agent-a7131b2ebdff5f551' into worktree-giggly-marinating-lamport
This commit is contained in:
commit
f970a7e5ce
19 files changed
+1743
-21
No files matched your search
@@ -0,0 +1,49 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
|
||||
|
||||
// TestRecoveryImportMovesTheInstallOnce drives the import the way an operator
|
||||
// does: the configured db_path locates every legacy file, and a second run
|
||||
// against the same database is refused because the first one completed.
|
||||
func TestRecoveryImportMovesTheInstallOnce(t *testing.T) {
|
||||
dbPath := filepath.Join(t.TempDir(), "archie.db")
|
||||
st := openTaskStore(t, dbPath+"-tasks.sqlite")
|
||||
seedTask(t, st, "carried over")
|
||||
if err := st.Close(); err != nil {
|
||||
t.Fatalf("close task store: %v", err)
|
||||
}
|
||||
url := pgtest.URL(t)
|
||||
|
||||
code, stdout, stderr := runRecoveryCmd(t, "import", "-db-path", dbPath, "-database-url", url)
|
||||
if code != 0 {
|
||||
t.Fatalf("import exited %d; stderr = %q", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, "tasks=1") {
|
||||
t.Errorf("import summary %q does not report the imported task", stdout)
|
||||
}
|
||||
|
||||
pool, err := postgres.Open(t.Context(), url)
|
||||
if err != nil {
|
||||
t.Fatalf("postgres.Open: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
var title string
|
||||
if err := pool.QueryRow(t.Context(), "SELECT title FROM tasks").Scan(&title); err != nil || title != "carried over" {
|
||||
t.Fatalf("imported task title = %q, %v; want %q", title, err, "carried over")
|
||||
}
|
||||
|
||||
code, _, stderr = runRecoveryCmd(t, "import", "-db-path", dbPath, "-database-url", url)
|
||||
if code != 1 || !strings.Contains(stderr, "already holds a completed import") {
|
||||
t.Fatalf("second import exited %d, stderr %q; want 1 naming the completed import", code, stderr)
|
||||
}
|
||||
}
|
||||
@@ -40,7 +40,7 @@ func run() int {
|
||||
flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)")
|
||||
flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)")
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store <backup|restore|validate|rollback> -h\n\n")
|
||||
fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store <backup|restore|validate|rollback|import> -h\n\n")
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
flag.Parse()
|
||||
|
||||
@@ -31,6 +31,9 @@ owns it, while backup takes a consistent snapshot of a serving store.
|
||||
configuration the daemon would boot with
|
||||
rollback -db FILE -kind KIND replay an earlier revision of a stored
|
||||
[-revision N] resource through the ordinary replace
|
||||
import [-db-path PATH] copy the legacy SQLite stores named from
|
||||
[-database-url URL] db_path into an empty Postgres database,
|
||||
[-config FILE] once; refuses while any Archie service runs
|
||||
`
|
||||
|
||||
// runRecovery parses and performs one offline recovery command, reporting the
|
||||
@@ -44,7 +47,8 @@ func runRecovery(args []string, stdout, stderr io.Writer) int {
|
||||
command := args[0]
|
||||
options := archied.StateStoreRecoveryOptions{Operation: command}
|
||||
switch command {
|
||||
case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback:
|
||||
case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback,
|
||||
archied.RecoveryImport:
|
||||
default:
|
||||
fmt.Fprintf(stderr, "archie-state-store: unknown command %q\n\n", command)
|
||||
fmt.Fprint(stderr, recoveryUsage)
|
||||
@@ -59,9 +63,14 @@ func runRecovery(args []string, stdout, stderr io.Writer) int {
|
||||
flags.StringVar(&options.From, "from", "", "snapshot file restore reads")
|
||||
flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays")
|
||||
flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)")
|
||||
// Only validate asks a question about the process rather than the file, so
|
||||
// only validate takes the configuration the process would boot with.
|
||||
if command == archied.RecoveryValidate {
|
||||
// validate asks a question about the process rather than the file, and
|
||||
// import reads db_path and database_url from it, so only those two take
|
||||
// the configuration the process would boot with.
|
||||
if command == archied.RecoveryImport {
|
||||
flags.StringVar(&options.DBPath, "db-path", "", "configured db_path the legacy files are named from (default: the configuration's)")
|
||||
flags.StringVar(&options.DatabaseURL, "database-url", "", "Postgres URL to import into (default: the configuration's database_url)")
|
||||
}
|
||||
if command == archied.RecoveryValidate || command == archied.RecoveryImport {
|
||||
flags.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory the daemon boots with")
|
||||
flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
|
||||
}
|
||||
|
||||
@@ -356,7 +356,7 @@ func TestRecoveryValidateRunsTheGateBootRuns(t *testing.T) {
|
||||
// path in the same binary exits 0 for the same request, so a script probing the
|
||||
// recovery surface must not read a requested help as a failure.
|
||||
func TestRecoveryHelpExitsZero(t *testing.T) {
|
||||
commands := []string{archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback}
|
||||
commands := []string{archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback, archied.RecoveryImport}
|
||||
for _, command := range commands {
|
||||
code, _, stderr := runRecoveryCmd(t, command, "-h")
|
||||
if code != 0 {
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/legacyimport"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
|
||||
)
|
||||
|
||||
// legacySources names an install's three legacy SQLite files from its
|
||||
// configured db_path, the same derivation the legacy openers use, so an
|
||||
// operator cannot import one store and forget another.
|
||||
func legacySources(dbPath string) legacyimport.Sources {
|
||||
return legacyimport.Sources{
|
||||
StateStore: taskDBPath(dbPath),
|
||||
EDA: edaDBPath(dbPath),
|
||||
Gateway: conversationDBPath(dbPath),
|
||||
}
|
||||
}
|
||||
|
||||
// importLegacy runs the one-time import of the legacy files into Postgres.
|
||||
// It is an offline command of the State Store binary, not of archied: offline
|
||||
// maintenance of a store belongs to the process that owns it.
|
||||
func importLegacy(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
|
||||
dbPath, url := options.DBPath, options.DatabaseURL
|
||||
if dbPath == "" || url == "" {
|
||||
cfg, err := bootConfig(ctx, options)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if dbPath == "" {
|
||||
dbPath = cfg.DBPath
|
||||
}
|
||||
if url == "" {
|
||||
url = cfg.DatabaseURL
|
||||
}
|
||||
}
|
||||
if dbPath == "" {
|
||||
return "", errors.New("import requires db_path (-db-path or the configuration)")
|
||||
}
|
||||
if url == "" {
|
||||
return "", errors.New("import requires database_url (-database-url or the configuration)")
|
||||
}
|
||||
pool, err := postgres.Open(ctx, url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer pool.Close()
|
||||
report, err := legacyimport.Import(ctx, pool, legacySources(dbPath))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("imported %s into Postgres and recorded the completion: %s", dbPath, report), nil
|
||||
}
|
||||
@@ -46,6 +46,7 @@ const (
|
||||
RecoveryRestore = "restore"
|
||||
RecoveryValidate = "validate"
|
||||
RecoveryRollback = "rollback"
|
||||
RecoveryImport = "import"
|
||||
)
|
||||
|
||||
// StateStoreRecoveryOptions are the process inputs for one offline recovery
|
||||
@@ -71,6 +72,11 @@ type StateStoreRecoveryOptions struct {
|
||||
// Revision is the revision rollback replays. Zero means the newest
|
||||
// revision older than the one the resource carries now.
|
||||
Revision int64
|
||||
// DBPath and DatabaseURL are import's source and target: the configured
|
||||
// db_path every legacy file is named from, and the Postgres URL. Either
|
||||
// left empty is read from the configuration.
|
||||
DBPath string
|
||||
DatabaseURL string
|
||||
}
|
||||
|
||||
// RunStateStoreRecovery performs one offline operation on the task database and
|
||||
@@ -81,18 +87,13 @@ func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOption
|
||||
if options.DB == "" || options.Out == "" {
|
||||
return "", errors.New("backup requires -db and -out")
|
||||
}
|
||||
if err := store.Backup(ctx, options.DB, options.Out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("backed up %s to %s", options.DB, options.Out), nil
|
||||
return summarise(store.Backup(ctx, options.DB, options.Out), "backed up %s to %s", options.DB, options.Out)
|
||||
case RecoveryRestore:
|
||||
if options.DB == "" || options.From == "" {
|
||||
return "", errors.New("restore requires -db and -from")
|
||||
}
|
||||
if err := store.Restore(ctx, options.DB, options.From); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("restored %s from %s; start the State Store again", options.DB, options.From), nil
|
||||
return summarise(store.Restore(ctx, options.DB, options.From),
|
||||
"restored %s from %s; start the State Store again", options.DB, options.From)
|
||||
case RecoveryValidate:
|
||||
if options.DB == "" {
|
||||
return "", errors.New("validate requires -db")
|
||||
@@ -103,11 +104,21 @@ func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOption
|
||||
return "", errors.New("rollback requires -db and -kind")
|
||||
}
|
||||
return rollbackResource(ctx, options)
|
||||
case RecoveryImport:
|
||||
return importLegacy(ctx, options)
|
||||
default:
|
||||
return "", fmt.Errorf("unknown recovery command %q", options.Operation)
|
||||
}
|
||||
}
|
||||
|
||||
// summarise is an operation's stdout line, or its error.
|
||||
func summarise(err error, format string, args ...any) (string, error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf(format, args...), nil
|
||||
}
|
||||
|
||||
// validateStore answers the question an operator actually has: would archied
|
||||
// start against this file. It runs the checks that stop it, in the order boot
|
||||
// meets them.
|
||||
|
||||
@@ -0,0 +1,387 @@
|
||||
// Package legacyimport moves an existing install's SQLite stores (the State
|
||||
// Store's task database, the PocketBase event store and the Gateway
|
||||
// conversation store) into Postgres, once.
|
||||
//
|
||||
// The import is one transaction: it refuses a target that already holds data
|
||||
// in a domain it imports, refuses rather than skips a row the target cannot
|
||||
// hold, verifies what it wrote against what it read, and only then records a
|
||||
// completion row and commits. A refusal at any point leaves the target as it
|
||||
// was, migrated and empty, so a failed import is never serveable.
|
||||
//
|
||||
// It reads through legacyread, which never writes to the source files, and is
|
||||
// deleted together with it.
|
||||
package legacyimport
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/legacyread"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb"
|
||||
)
|
||||
|
||||
// ErrRefused wraps every refusal. The message names each offending table,
|
||||
// row, value and rule.
|
||||
var ErrRefused = errors.New("legacy import refused")
|
||||
|
||||
// Sources are the legacy files. A path whose file does not exist means that
|
||||
// domain has nothing to import.
|
||||
type Sources struct {
|
||||
StateStore string
|
||||
EDA string
|
||||
Gateway string
|
||||
}
|
||||
|
||||
type domain struct {
|
||||
name string
|
||||
path string
|
||||
tables []legacyread.Table
|
||||
}
|
||||
|
||||
func (s Sources) domains() []domain {
|
||||
return []domain{
|
||||
{"state-store", s.StateStore, legacyread.StateStoreTables},
|
||||
{"eda", s.EDA, legacyread.EDATables},
|
||||
{"gateway", s.Gateway, legacyread.GatewayTables},
|
||||
}
|
||||
}
|
||||
|
||||
// present returns the domains whose source file exists.
|
||||
func (s Sources) present() ([]domain, error) {
|
||||
var out []domain
|
||||
for _, d := range s.domains() {
|
||||
if d.path == "" {
|
||||
continue
|
||||
}
|
||||
_, err := os.Stat(d.path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacy import: %w", err)
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// HasLegacyData reports whether any legacy source holds a row. With
|
||||
// postgres.ImportComplete it is what decides whether a Postgres database may
|
||||
// serve: legacy data without a completed import must be refused.
|
||||
func HasLegacyData(ctx context.Context, src Sources) (bool, error) {
|
||||
for _, d := range src.domains() {
|
||||
if d.path == "" {
|
||||
continue
|
||||
}
|
||||
has, err := legacyread.HasData(ctx, d.path, d.tables)
|
||||
if err != nil || has {
|
||||
return has, err
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Report is what an import wrote.
|
||||
type Report struct {
|
||||
// Rows is each imported table's row count.
|
||||
Rows map[string]int
|
||||
// UnindexedMessages are Gateway message ids the source's FTS5 index did
|
||||
// not cover. The target derives its search column from every row, so
|
||||
// these become searchable; the count says the source index was stale.
|
||||
UnindexedMessages []string
|
||||
}
|
||||
|
||||
func (r Report) String() string {
|
||||
var b strings.Builder
|
||||
for _, table := range slices.Sorted(maps.Keys(r.Rows)) {
|
||||
fmt.Fprintf(&b, "%s=%d ", table, r.Rows[table])
|
||||
}
|
||||
fmt.Fprintf(&b, "unindexed_messages=%d", len(r.UnindexedMessages))
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// refusal collects every problem found before giving up, so an operator fixes
|
||||
// a source in one pass rather than one row per attempt.
|
||||
type refusal []string
|
||||
|
||||
func (r *refusal) add(format string, args ...any) { *r = append(*r, fmt.Sprintf(format, args...)) }
|
||||
|
||||
func (r *refusal) err() error {
|
||||
if len(*r) == 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%w:\n %s", ErrRefused, strings.Join(*r, "\n "))
|
||||
}
|
||||
|
||||
// Import runs the one-time import from src into the database behind pool.
|
||||
// Every service that serves the database must be stopped: the import holds
|
||||
// each service's ownership claim for its whole run and refuses if any is
|
||||
// taken.
|
||||
func Import(ctx context.Context, pool *pgxpool.Pool, src Sources) (Report, error) {
|
||||
present, err := src.present()
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
if len(present) == 0 {
|
||||
return Report{}, fmt.Errorf("%w: no legacy source files found (%s, %s, %s)", ErrRefused, src.StateStore, src.EDA, src.Gateway)
|
||||
}
|
||||
before, err := hashSources(present)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
if err := postgres.Migrate(ctx, pool, postgres.Migrations()); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
release, err := claimServices(ctx, pool)
|
||||
defer release()
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
sources, closeSources, err := openSources(ctx, present)
|
||||
defer closeSources()
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
|
||||
var problems refusal
|
||||
if err := checkSources(ctx, sources, &problems); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
if err := problems.err(); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return Report{}, fmt.Errorf("legacy import: begin: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
|
||||
report, err := importInto(ctx, tx, present, sources, &problems)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
if err := checkUnchanged(before, present, &problems); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
if err := problems.err(); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
if err := complete(ctx, tx, present, report); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// checkUnchanged refuses when a source's bytes moved during the import: a
|
||||
// writer that was not stopped would leave the target short of its last writes.
|
||||
func checkUnchanged(before map[string][32]byte, present []domain, problems *refusal) error {
|
||||
after, err := hashSources(present)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for path, sum := range before {
|
||||
if after[path] != sum {
|
||||
problems.add("%s changed while it was read: stop the process writing it and import again", path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// complete writes the completion record and commits, so the record exists
|
||||
// exactly when the imported rows do.
|
||||
func complete(ctx context.Context, tx pgx.Tx, present []domain, report Report) error {
|
||||
paths := make([]string, len(present))
|
||||
for i, d := range present {
|
||||
paths[i] = d.path
|
||||
}
|
||||
if err := postgresdb.New(tx).InsertImportCompletion(ctx, postgresdb.InsertImportCompletionParams{
|
||||
Sources: strings.Join(paths, "\n"),
|
||||
Report: report.String(),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("legacy import: completion record: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("legacy import: commit: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// claimServices takes every service's ownership claim without waiting. The
|
||||
// returned release is safe to call whether or not the claims were taken.
|
||||
func claimServices(ctx context.Context, pool *pgxpool.Pool) (func(), error) {
|
||||
var claims []*postgres.Ownership
|
||||
release := func() {
|
||||
for _, c := range claims {
|
||||
_ = c.Release(context.WithoutCancel(ctx))
|
||||
}
|
||||
}
|
||||
for _, name := range []string{postgres.OwnerStateStore, postgres.OwnerGateway} {
|
||||
claim, err := postgres.AcquireOwnership(ctx, pool, name)
|
||||
if err != nil {
|
||||
return release, fmt.Errorf("legacy import: stop every Archie service first: %w", err)
|
||||
}
|
||||
claims = append(claims, claim)
|
||||
}
|
||||
return release, nil
|
||||
}
|
||||
|
||||
// openSources opens each present source read-only, keyed by domain name.
|
||||
func openSources(ctx context.Context, present []domain) (map[string]*legacyread.Source, func(), error) {
|
||||
sources := map[string]*legacyread.Source{}
|
||||
closeAll := func() {
|
||||
for _, s := range sources {
|
||||
_ = s.Close()
|
||||
}
|
||||
}
|
||||
for _, d := range present {
|
||||
s, err := legacyread.Open(ctx, d.path)
|
||||
if err != nil {
|
||||
return nil, closeAll, err
|
||||
}
|
||||
sources[d.name] = s
|
||||
}
|
||||
return sources, closeAll, nil
|
||||
}
|
||||
|
||||
// importInto loads and verifies every present domain inside tx, adding to
|
||||
// problems anything that must stop the commit.
|
||||
func importInto(ctx context.Context, tx pgx.Tx, present []domain, sources map[string]*legacyread.Source, problems *refusal) (Report, error) {
|
||||
if err := checkTarget(ctx, tx, present, problems); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
if err := problems.err(); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
report := Report{Rows: map[string]int{}}
|
||||
var loaded []loadedTable
|
||||
for _, d := range present {
|
||||
for _, t := range d.tables {
|
||||
lt, err := load(ctx, tx, sources[d.name], t, problems)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
loaded = append(loaded, lt)
|
||||
report.Rows[t.Name] = len(lt.keys)
|
||||
}
|
||||
}
|
||||
if err := problems.err(); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
for _, lt := range loaded {
|
||||
if err := verify(ctx, tx, lt, problems); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
}
|
||||
if gw := sources["gateway"]; gw != nil {
|
||||
var err error
|
||||
if report.UnindexedMessages, err = unindexedMessages(ctx, gw, loaded); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// hashSources fingerprints each source and its WAL, so a writer that was not
|
||||
// stopped is caught rather than assumed away.
|
||||
func hashSources(present []domain) (map[string][32]byte, error) {
|
||||
out := map[string][32]byte{}
|
||||
for _, d := range present {
|
||||
for _, path := range []string{d.path, d.path + "-wal"} {
|
||||
f, err := os.Open(path)
|
||||
if errors.Is(err, os.ErrNotExist) && path != d.path {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacy import: %w", err)
|
||||
}
|
||||
h := sha256.New()
|
||||
_, err = io.Copy(h, f)
|
||||
_ = f.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacy import: hash %s: %w", path, err)
|
||||
}
|
||||
out[path] = [32]byte(h.Sum(nil))
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// checkSources finds the source rows Postgres would reject or reorder.
|
||||
func checkSources(ctx context.Context, sources map[string]*legacyread.Source, problems *refusal) error {
|
||||
if s := sources["state-store"]; s != nil {
|
||||
for _, fk := range legacyread.IdentityForeignKeys {
|
||||
orphans, err := s.Orphans(ctx, fk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, o := range orphans {
|
||||
problems.add("%s %s: %s %q has no row in %s (foreign key)", fk.Table, o.Key, fk.Column, o.Parent, fk.Parent)
|
||||
}
|
||||
}
|
||||
aliases, err := s.DuplicateAliases(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, d := range aliases {
|
||||
problems.add("identity_aliases %s: aliases collide on unique lower(alias) %q", strings.Join(d.Values, ", "), d.Key)
|
||||
}
|
||||
collisions, err := s.MicrosecondCollisions(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, c := range collisions {
|
||||
problems.add("events %d and %d: at differs only below a microsecond (%s) while ids run the other way, so (at, id) order would change",
|
||||
c.EarlierID, c.LaterID, c.At.Format("2006-01-02T15:04:05.000000Z"))
|
||||
}
|
||||
}
|
||||
if s := sources["eda"]; s != nil {
|
||||
dups, err := s.DuplicateBindings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, d := range dups {
|
||||
problems.add("bindings %s: more than one binding for source %q (unique source); resolve them and import again",
|
||||
strings.Join(d.Values, ", "), d.Key)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkTarget refuses a database that already completed an import, or that
|
||||
// holds rows in a table owned by a domain being imported. The check is an
|
||||
// allowlist of those tables, so goose's version table and other domains'
|
||||
// tables never count.
|
||||
func checkTarget(ctx context.Context, tx pgx.Tx, present []domain, problems *refusal) error {
|
||||
done, err := postgresdb.New(tx).ImportCompleted(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("legacy import: completion record: %w", err)
|
||||
}
|
||||
if done {
|
||||
problems.add("this database already holds a completed import")
|
||||
return nil
|
||||
}
|
||||
for _, d := range present {
|
||||
for _, t := range d.tables {
|
||||
var rows bool
|
||||
if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM "+pgx.Identifier{t.Name}.Sanitize()+")").Scan(&rows); err != nil {
|
||||
return fmt.Errorf("legacy import: %s: %w", t.Name, err)
|
||||
}
|
||||
if rows {
|
||||
problems.add("%s is not empty: the %s target already holds data", t.Name, d.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,517 @@
|
||||
package legacyimport
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) }
|
||||
|
||||
// The fixtures apply the legacy schemas as their production openers leave
|
||||
// them, columns added by ALTER TABLE included, with values in the layouts the
|
||||
// production writers used.
|
||||
const stateStoreDDL = `
|
||||
CREATE TABLE tasks (id INTEGER PRIMARY KEY AUTOINCREMENT, owner TEXT NOT NULL, repo TEXT NOT NULL,
|
||||
issue_number INTEGER NOT NULL, title TEXT DEFAULT '', body TEXT DEFAULT '', labels TEXT DEFAULT '',
|
||||
status TEXT DEFAULT 'queued', workflow TEXT DEFAULT '', stage TEXT DEFAULT '', branch TEXT DEFAULT '',
|
||||
plan TEXT DEFAULT '', notes TEXT DEFAULT '', pr_number INTEGER DEFAULT 0, tokens_used INTEGER DEFAULT 0,
|
||||
iterations INTEGER DEFAULT 0, attempt INTEGER DEFAULT 0, park_reason TEXT DEFAULT '',
|
||||
watch_comment_id INTEGER DEFAULT 0, park_class TEXT DEFAULT 'needs_human', remediation_rounds INTEGER DEFAULT 0,
|
||||
retry_count INTEGER DEFAULT 0, source TEXT DEFAULT 'forge', identity TEXT DEFAULT '', binding_id TEXT DEFAULT '',
|
||||
binding_version INTEGER DEFAULT 0, review_payload TEXT DEFAULT '', workflow_definition_version INTEGER DEFAULT 0,
|
||||
workflow_definition_digest TEXT DEFAULT '', workflow_definition_yaml TEXT DEFAULT '',
|
||||
created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')));
|
||||
ALTER TABLE tasks ADD COLUMN review_cursor INTEGER NOT NULL DEFAULT 0;
|
||||
CREATE TABLE transitions (id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL,
|
||||
at TEXT DEFAULT (datetime('now')), from_status TEXT NOT NULL, to_status TEXT NOT NULL, detail TEXT DEFAULT '');
|
||||
CREATE TABLE events (id INTEGER PRIMARY KEY AUTOINCREMENT, at TEXT NOT NULL, kind TEXT NOT NULL,
|
||||
task_id INTEGER DEFAULT 0, repo TEXT DEFAULT '', issue INTEGER DEFAULT 0, workflow TEXT DEFAULT '',
|
||||
stage TEXT DEFAULT '', attempt INTEGER DEFAULT 0, actor_id TEXT DEFAULT '', actor_kind TEXT DEFAULT '',
|
||||
principal_id TEXT DEFAULT '', detail TEXT DEFAULT '', data TEXT DEFAULT '{}');
|
||||
CREATE TABLE resources (kind TEXT PRIMARY KEY, value BLOB NOT NULL, version INTEGER NOT NULL, updated_at TEXT NOT NULL);
|
||||
CREATE TABLE resource_history (id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, value BLOB NOT NULL,
|
||||
version INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL, request_id TEXT NOT NULL,
|
||||
expected_version INTEGER NOT NULL, current_version INTEGER NOT NULL, at TEXT NOT NULL);
|
||||
CREATE TABLE channel_status (id TEXT PRIMARY KEY, name TEXT DEFAULT '', state TEXT DEFAULT '', detail TEXT DEFAULT '',
|
||||
configured INTEGER NOT NULL DEFAULT 0, reload_supported INTEGER NOT NULL DEFAULT 0, observed_at TEXT NOT NULL);
|
||||
CREATE TABLE apply_status (process TEXT NOT NULL, kind TEXT NOT NULL, applied_version INTEGER DEFAULT 0,
|
||||
error TEXT DEFAULT '', reported_at TEXT NOT NULL, PRIMARY KEY (process, kind));
|
||||
CREATE TABLE config_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), schema TEXT DEFAULT '', document TEXT DEFAULT '',
|
||||
published_at TEXT NOT NULL);
|
||||
CREATE TABLE identities (id TEXT PRIMARY KEY, kind TEXT NOT NULL, display_name TEXT NOT NULL, lifecycle TEXT NOT NULL,
|
||||
version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL);
|
||||
CREATE TABLE identity_aliases (alias TEXT PRIMARY KEY COLLATE NOCASE, identity_id TEXT NOT NULL REFERENCES identities(id));
|
||||
CREATE TABLE identity_events (id INTEGER PRIMARY KEY AUTOINCREMENT, identity_id TEXT NOT NULL, event_type TEXT NOT NULL,
|
||||
from_lifecycle TEXT NOT NULL, to_lifecycle TEXT NOT NULL, display_name TEXT NOT NULL, actor_id TEXT NOT NULL,
|
||||
source TEXT NOT NULL, request_id TEXT NOT NULL UNIQUE, at TEXT NOT NULL);
|
||||
CREATE TABLE identity_subjects (issuer TEXT NOT NULL, subject TEXT NOT NULL,
|
||||
identity_id TEXT NOT NULL REFERENCES identities(id), bound_at TEXT NOT NULL, PRIMARY KEY (issuer, subject));
|
||||
|
||||
INSERT INTO tasks (id, owner, repo, issue_number, review_cursor, created_at, updated_at) VALUES
|
||||
(1, 'o', 'r', 7, 3, '2026-01-01 00:00:00', '2026-01-01 00:00:01'),
|
||||
(4, 'o', 'r', 1000000000000000, 0, '2026-01-02 00:00:00', '2026-01-02 00:00:00'),
|
||||
(9, 'o', 'r', 8, 0, '2026-01-02 00:00:00', '2026-01-02 00:00:00');
|
||||
DELETE FROM tasks WHERE id = 9;
|
||||
INSERT INTO transitions (task_id, at, from_status, to_status) VALUES (1, '2026-01-01 00:00:00', 'queued', 'running');
|
||||
INSERT INTO events (id, at, kind, data) VALUES
|
||||
(1, '2026-01-01T00:00:00.000000100Z', 'a', 'null'),
|
||||
(2, '2026-01-01T00:00:01.000000000Z', 'b', '{"k":1}'),
|
||||
(5, '2026-01-01T00:00:00.500000000Z', 'c', '{}');
|
||||
INSERT INTO resources VALUES ('config', X'00ff10', 2, '2026-01-01T00:00:00.123456789Z');
|
||||
INSERT INTO resource_history (kind, value, version, actor, source, request_id, expected_version, current_version, at)
|
||||
VALUES ('config', X'00ff10', 2, 'sam', 'ui', 'req-1', 1, 1, '2026-01-01T00:00:00Z');
|
||||
INSERT INTO channel_status VALUES ('tg', 'Telegram', 'up', '', 1, 0, '2026-01-01T00:00:00Z');
|
||||
INSERT INTO apply_status VALUES ('daemon', 'config', 2, '', '2026-01-01T00:00:00Z');
|
||||
INSERT INTO config_snapshot VALUES (1, 's', '{}', '2026-01-01T00:00:00Z');
|
||||
INSERT INTO identities VALUES ('id-1', 'agent', 'Archie', 'active', 1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z');
|
||||
INSERT INTO identity_aliases VALUES ('archie', 'id-1');
|
||||
INSERT INTO identity_events (identity_id, event_type, from_lifecycle, to_lifecycle, display_name, actor_id, source, request_id, at)
|
||||
VALUES ('id-1', 'created', '', 'active', 'Archie', 'sam', 'boot', 'rq-1', '2026-01-01T00:00:00Z');
|
||||
INSERT INTO identity_subjects VALUES ('https://issuer', 'sub-1', 'id-1', '2026-01-01T00:00:00Z');
|
||||
`
|
||||
|
||||
// secret is an encrypted binding secret envelope; it must arrive unchanged.
|
||||
const secret = "arcie-binding:v1:0123456789abcdef:AbC-_dEf012"
|
||||
|
||||
const edaDDL = `
|
||||
CREATE TABLE captures (id TEXT PRIMARY KEY NOT NULL, authenticated BOOLEAN DEFAULT FALSE NOT NULL,
|
||||
body TEXT DEFAULT '' NOT NULL, content_type TEXT DEFAULT '' NOT NULL, headers JSON DEFAULT NULL,
|
||||
received_at TEXT DEFAULT '' NOT NULL, remote_addr TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL);
|
||||
CREATE TABLE mappings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, fields JSON DEFAULT NULL,
|
||||
name TEXT DEFAULT '' NOT NULL, source_hint TEXT DEFAULT '' NOT NULL, updated_at TEXT DEFAULT '' NOT NULL);
|
||||
CREATE TABLE bindings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, mapping TEXT DEFAULT '' NOT NULL,
|
||||
name TEXT DEFAULT '' NOT NULL, owner TEXT DEFAULT '' NOT NULL, repo TEXT DEFAULT '' NOT NULL,
|
||||
secret TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL, status TEXT DEFAULT '' NOT NULL,
|
||||
updated_at TEXT DEFAULT '' NOT NULL, version NUMERIC DEFAULT 0 NOT NULL, workflow TEXT DEFAULT '' NOT NULL);
|
||||
CREATE TABLE binding_dispatches (id TEXT PRIMARY KEY NOT NULL, binding TEXT DEFAULT '' NOT NULL,
|
||||
binding_version NUMERIC DEFAULT 0 NOT NULL, capture TEXT DEFAULT '' NOT NULL, dispatched_at TEXT DEFAULT '' NOT NULL,
|
||||
task_id NUMERIC DEFAULT 0 NOT NULL);
|
||||
CREATE UNIQUE INDEX idx_binding_dispatch_once ON binding_dispatches (binding, capture);
|
||||
CREATE TABLE playbook_dispatches (id TEXT PRIMARY KEY NOT NULL, action_id TEXT DEFAULT '' NOT NULL,
|
||||
dispatched_at TEXT DEFAULT '' NOT NULL, event_id TEXT DEFAULT '' NOT NULL, playbook_id TEXT DEFAULT '' NOT NULL,
|
||||
playbook_version TEXT DEFAULT '' NOT NULL);
|
||||
CREATE TABLE tool_calls (id TEXT PRIMARY KEY NOT NULL, args JSON DEFAULT NULL, attempt NUMERIC DEFAULT 0 NOT NULL,
|
||||
called_at TEXT DEFAULT '' NOT NULL, duration_ms NUMERIC DEFAULT 0 NOT NULL, error TEXT DEFAULT '' NOT NULL,
|
||||
result JSON DEFAULT NULL, task_id NUMERIC DEFAULT 0 NOT NULL, tool TEXT DEFAULT '' NOT NULL);
|
||||
|
||||
INSERT INTO captures VALUES ('rcap1', 1, '{}', 'application/json', '{"x":["1"]}', '2026-01-01T00:00:00.000000000Z', '127.0.0.1', 'gh');
|
||||
INSERT INTO captures VALUES ('rcap2', 0, '', '', NULL, '2026-01-01T00:00:01.000000000Z', '', 'gh');
|
||||
INSERT INTO mappings VALUES ('rmap1', '2026-01-01 00:00:00.000Z', '[]', 'm', 'gh', '2026-01-01 00:00:00.000Z');
|
||||
INSERT INTO bindings VALUES ('rbind1', '2026-01-01 00:00:00.000Z', 'rmap1', 'b', 'o', 'r', '` + secret + `', 'gh', 'armed', '2026-01-01 00:00:00.000Z', 1, 'implement');
|
||||
INSERT INTO binding_dispatches VALUES ('rdisp1', 'rbind1', 1, 'rcap1', '2026-01-01 00:00:00.000Z', 4);
|
||||
INSERT INTO playbook_dispatches VALUES ('rpb1', 'act', '2026-01-01 00:00:00.000Z', 'ev', 'pb', 'v1');
|
||||
INSERT INTO tool_calls VALUES ('rtool1', NULL, 1, '2026-01-01 00:00:00.000Z', 12, '', '"exit 0"', 4, 'bash');
|
||||
`
|
||||
|
||||
const gatewayDDL = `
|
||||
CREATE TABLE sessions (session_id TEXT PRIMARY KEY, platform TEXT NOT NULL DEFAULT '', bot_user TEXT NOT NULL DEFAULT '',
|
||||
channel_id TEXT NOT NULL DEFAULT '', thread_id TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '',
|
||||
parent_session_id TEXT NOT NULL DEFAULT '', branch_name TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL DEFAULT 0, last_active_at INTEGER NOT NULL DEFAULT 0);
|
||||
CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT NOT NULL, session_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL DEFAULT '', sender TEXT NOT NULL DEFAULT '', text TEXT NOT NULL DEFAULT '', ts INTEGER NOT NULL,
|
||||
UNIQUE(session_id, message_id));
|
||||
ALTER TABLE messages ADD COLUMN role TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE messages ADD COLUMN sender_id TEXT NOT NULL DEFAULT '';
|
||||
CREATE VIRTUAL TABLE messages_fts USING fts5(sender, text, content='messages', content_rowid='id');
|
||||
CREATE TRIGGER messages_ai AFTER INSERT ON messages BEGIN
|
||||
INSERT INTO messages_fts(rowid, sender, text) VALUES (new.id, new.sender, new.text);
|
||||
END;
|
||||
CREATE TABLE turns (turn_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, source_id TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL, attempt INTEGER NOT NULL DEFAULT 0, owner_id TEXT NOT NULL DEFAULT '',
|
||||
input_message_id TEXT NOT NULL DEFAULT '', assistant_message_id TEXT NOT NULL DEFAULT '',
|
||||
partial_text TEXT NOT NULL DEFAULT '', response_text TEXT NOT NULL DEFAULT '', tool_calls TEXT NOT NULL DEFAULT '[]',
|
||||
error TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL);
|
||||
|
||||
INSERT INTO sessions VALUES ('s1', 'telegram', 'bot', 'c1', '', 'Chat', '', '', 1000, 2000);
|
||||
INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m1', 's1', 'sam', 'hello there', 1000, 'user');
|
||||
INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m2', 's1', 'archie', 'hi', 1001, 'assistant');
|
||||
INSERT INTO turns VALUES ('t1', 's1', '', 'done', 1, '', 'm1', 'm2', '', 'hi', '[]', '', 1000, 1001);
|
||||
`
|
||||
|
||||
// install writes the three legacy files the way an existing install holds
|
||||
// them and returns their paths. mutate runs extra statements against a file
|
||||
// before the import sees it.
|
||||
func install(t *testing.T, mutate map[string]string) Sources {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
src := Sources{
|
||||
StateStore: filepath.Join(dir, "archie.db-tasks.sqlite"),
|
||||
EDA: filepath.Join(dir, "archie.db-eda.sqlite"),
|
||||
Gateway: filepath.Join(dir, "archie.db-conversations.sqlite"),
|
||||
}
|
||||
for path, ddl := range map[string]string{src.StateStore: stateStoreDDL, src.EDA: edaDDL, src.Gateway: gatewayDDL} {
|
||||
sqliteExec(t, path, ddl)
|
||||
}
|
||||
for path, stmt := range mutate {
|
||||
switch path {
|
||||
case "state":
|
||||
sqliteExec(t, src.StateStore, stmt)
|
||||
case "eda":
|
||||
sqliteExec(t, src.EDA, stmt)
|
||||
case "gateway":
|
||||
sqliteExec(t, src.Gateway, stmt)
|
||||
}
|
||||
}
|
||||
return src
|
||||
}
|
||||
|
||||
func sqliteExec(t *testing.T, path, stmt string) {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
t.Fatalf("open fixture: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
if _, err := db.ExecContext(t.Context(), stmt); err != nil {
|
||||
t.Fatalf("fixture %s: %v", filepath.Base(path), err)
|
||||
}
|
||||
}
|
||||
|
||||
func hashFiles(t *testing.T, dir string) map[string][32]byte {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read dir: %v", err)
|
||||
}
|
||||
out := map[string][32]byte{}
|
||||
for _, e := range entries {
|
||||
b, err := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", e.Name(), err)
|
||||
}
|
||||
out[e.Name()] = sha256.Sum256(b)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func target(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
pool, err := postgres.Open(t.Context(), pgtest.URL(t))
|
||||
if err != nil {
|
||||
t.Fatalf("postgres.Open: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
return pool
|
||||
}
|
||||
|
||||
func migrated(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
pool := target(t)
|
||||
if err := postgres.Migrate(t.Context(), pool, postgres.Migrations()); err != nil {
|
||||
t.Fatalf("Migrate: %v", err)
|
||||
}
|
||||
return pool
|
||||
}
|
||||
|
||||
func count(t *testing.T, pool *pgxpool.Pool, table string) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := pool.QueryRow(t.Context(), "SELECT count(*) FROM "+table).Scan(&n); err != nil {
|
||||
t.Fatalf("count %s: %v", table, err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestImportCopiesEverySourceAndLeavesThemUntouched(t *testing.T) {
|
||||
src := install(t, nil)
|
||||
before := hashFiles(t, filepath.Dir(src.StateStore))
|
||||
pool := target(t)
|
||||
|
||||
report, err := Import(t.Context(), pool, src)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
if after := hashFiles(t, filepath.Dir(src.StateStore)); len(after) != len(before) {
|
||||
t.Fatalf("source directory changed: %d files before, %d after", len(before), len(after))
|
||||
} else {
|
||||
for name, sum := range before {
|
||||
if after[name] != sum {
|
||||
t.Errorf("source %s changed during import", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
done, err := postgres.ImportComplete(t.Context(), pool)
|
||||
if err != nil || !done {
|
||||
t.Fatalf("ImportComplete = %v, %v; want true", done, err)
|
||||
}
|
||||
|
||||
counts := map[string]int{
|
||||
"tasks": 2, "transitions": 1, "events": 3, "resources": 1, "resource_history": 1,
|
||||
"channel_status": 1, "apply_status": 1, "config_snapshot": 1, "identities": 1,
|
||||
"identity_aliases": 1, "identity_events": 1, "identity_subjects": 1,
|
||||
"captures": 2, "mappings": 1, "bindings": 1, "binding_dispatches": 1, "playbook_dispatches": 1,
|
||||
"tool_calls": 1, "sessions": 1, "messages": 2, "turns": 1,
|
||||
}
|
||||
for table, want := range counts {
|
||||
if got := count(t, pool, table); got != want {
|
||||
t.Errorf("%s rows = %d, want %d", table, got, want)
|
||||
}
|
||||
if got := report.Rows[table]; got != want {
|
||||
t.Errorf("report rows[%s] = %d, want %d", table, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
ctx := t.Context()
|
||||
var (
|
||||
reviewCursor int64
|
||||
value []byte
|
||||
gotSecret string
|
||||
headers string
|
||||
result string
|
||||
configured bool
|
||||
)
|
||||
checks := []struct {
|
||||
query string
|
||||
dest any
|
||||
}{
|
||||
{"SELECT review_cursor FROM tasks WHERE id = 1", &reviewCursor},
|
||||
{"SELECT value FROM resource_history WHERE id = 1", &value},
|
||||
{"SELECT secret FROM bindings WHERE id = 'rbind1'", &gotSecret},
|
||||
{"SELECT headers FROM captures WHERE id = 'rcap2'", &headers},
|
||||
{"SELECT result FROM tool_calls WHERE id = 'rtool1'", &result},
|
||||
{"SELECT configured FROM channel_status WHERE id = 'tg'", &configured},
|
||||
}
|
||||
for _, c := range checks {
|
||||
if err := pool.QueryRow(ctx, c.query).Scan(c.dest); err != nil {
|
||||
t.Fatalf("%s: %v", c.query, err)
|
||||
}
|
||||
}
|
||||
if reviewCursor != 3 {
|
||||
t.Errorf("review_cursor = %d, want 3", reviewCursor)
|
||||
}
|
||||
if !bytes.Equal(value, []byte{0x00, 0xff, 0x10}) {
|
||||
t.Errorf("resource_history.value = %x, want 00ff10", value)
|
||||
}
|
||||
if gotSecret != secret {
|
||||
t.Errorf("binding secret = %q, want %q", gotSecret, secret)
|
||||
}
|
||||
if headers != "" || result != "exit 0" || !configured {
|
||||
t.Errorf("headers %q, result %q, configured %v; want \"\", \"exit 0\", true", headers, result, configured)
|
||||
}
|
||||
|
||||
// Event order is (at, id) as the source had it, and the next event id
|
||||
// follows AUTOINCREMENT's high-water mark rather than reusing one.
|
||||
rows, err := pool.Query(ctx, "SELECT id FROM events ORDER BY at, id")
|
||||
if err != nil {
|
||||
t.Fatalf("events: %v", err)
|
||||
}
|
||||
var order []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
order = append(order, id)
|
||||
}
|
||||
if want := []int64{1, 5, 2}; !equalInts(order, want) {
|
||||
t.Errorf("event order = %v, want %v", order, want)
|
||||
}
|
||||
nextIDs := map[string]int64{"tasks": 10, "events": 6, "messages": 3}
|
||||
for table, want := range nextIDs {
|
||||
var got int64
|
||||
if err := pool.QueryRow(ctx, "SELECT nextval(pg_get_serial_sequence($1, 'id'))", table).Scan(&got); err != nil {
|
||||
t.Fatalf("nextval %s: %v", table, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("next %s id = %d, want %d", table, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
var hits int
|
||||
if err := pool.QueryRow(ctx, "SELECT count(*) FROM messages WHERE search @@ to_tsquery('simple', 'hello')").Scan(&hits); err != nil {
|
||||
t.Fatalf("search: %v", err)
|
||||
}
|
||||
if hits != 1 {
|
||||
t.Errorf("search hits for an imported message = %d, want 1", hits)
|
||||
}
|
||||
}
|
||||
|
||||
func equalInts(a, b []int64) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestImportRefuses(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate map[string]string
|
||||
seed func(t *testing.T, pool *pgxpool.Pool)
|
||||
want []string
|
||||
is error
|
||||
}{
|
||||
{
|
||||
name: "non-empty target",
|
||||
seed: func(t *testing.T, pool *pgxpool.Pool) {
|
||||
mustExec(t, pool, "INSERT INTO transitions (task_id, from_status, to_status) VALUES (1, 'a', 'b')")
|
||||
},
|
||||
want: []string{"transitions", "not empty"},
|
||||
},
|
||||
{
|
||||
name: "completed import",
|
||||
seed: func(t *testing.T, pool *pgxpool.Pool) {
|
||||
mustExec(t, pool, "INSERT INTO import_completion (id, sources, report) VALUES (1, 's', 'r')")
|
||||
},
|
||||
want: []string{"already"},
|
||||
},
|
||||
{
|
||||
name: "live state store",
|
||||
seed: func(t *testing.T, pool *pgxpool.Pool) { holdOwnership(t, pool, postgres.OwnerStateStore) },
|
||||
want: []string{"state-store"},
|
||||
is: postgres.ErrOwned,
|
||||
},
|
||||
{
|
||||
name: "live gateway",
|
||||
seed: func(t *testing.T, pool *pgxpool.Pool) { holdOwnership(t, pool, postgres.OwnerGateway) },
|
||||
want: []string{"gateway"},
|
||||
is: postgres.ErrOwned,
|
||||
},
|
||||
{
|
||||
name: "orphaned alias",
|
||||
mutate: map[string]string{"state": "INSERT INTO identity_aliases VALUES ('ghost', 'id-404')"},
|
||||
want: []string{"identity_aliases", "ghost", "id-404"},
|
||||
},
|
||||
{
|
||||
name: "duplicate bindings",
|
||||
mutate: map[string]string{"eda": "INSERT INTO bindings VALUES ('rbind2', '2026-01-01 00:00:00.000Z', '', 'b2', 'o', 'r', '', 'gh', 'pending_approval', '2026-01-01 00:00:00.000Z', 1, 'implement')"},
|
||||
want: []string{"bindings", "gh", "rbind1", "rbind2"},
|
||||
},
|
||||
{
|
||||
name: "microsecond collision",
|
||||
mutate: map[string]string{"state": "INSERT INTO events (id, at, kind) VALUES (7, '2026-01-01T00:00:00.000000050Z', 'd')"},
|
||||
want: []string{"events", "7", "microsecond"},
|
||||
},
|
||||
{
|
||||
name: "unparseable timestamp",
|
||||
mutate: map[string]string{"eda": "UPDATE captures SET received_at = 'yesterday' WHERE id = 'rcap2'"},
|
||||
want: []string{"captures", "rcap2", "yesterday", "received_at"},
|
||||
},
|
||||
{
|
||||
name: "null in a required column",
|
||||
mutate: map[string]string{"state": "UPDATE tasks SET title = NULL WHERE id = 4"},
|
||||
want: []string{"tasks", "4", "title", "NULL"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
src := install(t, tt.mutate)
|
||||
pool := migrated(t)
|
||||
if tt.seed != nil {
|
||||
tt.seed(t, pool)
|
||||
}
|
||||
_, err := Import(t.Context(), pool, src)
|
||||
if err == nil {
|
||||
t.Fatal("Import succeeded, want a refusal")
|
||||
}
|
||||
if tt.is != nil && !errors.Is(err, tt.is) {
|
||||
t.Errorf("Import error %v, want %v", err, tt.is)
|
||||
}
|
||||
for _, w := range tt.want {
|
||||
if !strings.Contains(err.Error(), w) {
|
||||
t.Errorf("refusal %q does not name %q", err, w)
|
||||
}
|
||||
}
|
||||
// Nothing of the import survives a refusal: the store stays
|
||||
// unserveable rather than half-imported.
|
||||
if got := count(t, pool, "tasks"); got != 0 {
|
||||
t.Errorf("tasks rows after refusal = %d, want 0", got)
|
||||
}
|
||||
if got := count(t, pool, "import_completion"); tt.name != "completed import" && got != 0 {
|
||||
t.Errorf("completion records after refusal = %d, want 0", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mustExec(t *testing.T, pool *pgxpool.Pool, stmt string) {
|
||||
t.Helper()
|
||||
if _, err := pool.Exec(t.Context(), stmt); err != nil {
|
||||
t.Fatalf("%s: %v", stmt, err)
|
||||
}
|
||||
}
|
||||
|
||||
// holdOwnership claims name from a separate pool, as a live service would.
|
||||
func holdOwnership(t *testing.T, pool *pgxpool.Pool, name string) {
|
||||
t.Helper()
|
||||
other, err := pgxpool.New(t.Context(), pool.Config().ConnString())
|
||||
if err != nil {
|
||||
t.Fatalf("second pool: %v", err)
|
||||
}
|
||||
t.Cleanup(other.Close)
|
||||
claim, err := postgres.AcquireOwnership(t.Context(), other, name)
|
||||
if err != nil {
|
||||
t.Fatalf("claim %s: %v", name, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = claim.Release(t.Context()) })
|
||||
}
|
||||
|
||||
// TestSiblingDomainDoesNotBlock pins that "non-empty" is per domain: a
|
||||
// domain with no legacy source is not imported, so rows in its tables do not
|
||||
// refuse the others.
|
||||
func TestSiblingDomainDoesNotBlock(t *testing.T) {
|
||||
src := install(t, nil)
|
||||
src.Gateway = filepath.Join(t.TempDir(), "absent.sqlite")
|
||||
pool := migrated(t)
|
||||
mustExec(t, pool, "INSERT INTO sessions (session_id) VALUES ('live')")
|
||||
if _, err := Import(t.Context(), pool, src); err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
if got := count(t, pool, "tasks"); got != 2 {
|
||||
t.Fatalf("tasks rows = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportRefusesWithNoSources(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := Sources{StateStore: filepath.Join(dir, "a"), EDA: filepath.Join(dir, "b"), Gateway: filepath.Join(dir, "c")}
|
||||
if _, err := Import(t.Context(), migrated(t), src); err == nil || !strings.Contains(err.Error(), "no legacy") {
|
||||
t.Fatalf("Import = %v, want a refusal naming no legacy sources", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasLegacyData(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
absent := Sources{StateStore: filepath.Join(dir, "a"), EDA: filepath.Join(dir, "b"), Gateway: filepath.Join(dir, "c")}
|
||||
onlyGateway := absent
|
||||
onlyGateway.Gateway = install(t, nil).Gateway
|
||||
tests := []struct {
|
||||
name string
|
||||
src Sources
|
||||
want bool
|
||||
}{
|
||||
{"fresh install", absent, false},
|
||||
{"one source with rows", onlyGateway, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := HasLegacyData(t.Context(), tt.src)
|
||||
if err != nil {
|
||||
t.Fatalf("HasLegacyData: %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("HasLegacyData = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
package legacyimport
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/legacyread"
|
||||
)
|
||||
|
||||
// droppedColumns are legacy columns with no target column by design:
|
||||
// PocketBase gave every ledger row a random record id, and the Postgres
|
||||
// ledgers are keyed by their natural unique key instead.
|
||||
var droppedColumns = map[string]bool{
|
||||
"binding_dispatches.id": true,
|
||||
"playbook_dispatches.id": true,
|
||||
}
|
||||
|
||||
// targetKeys override a table's legacy key where the target has no column
|
||||
// for it.
|
||||
var targetKeys = map[string][]string{
|
||||
"binding_dispatches": {"binding", "capture"},
|
||||
"playbook_dispatches": {"playbook_id", "playbook_version", "event_id", "action_id"},
|
||||
}
|
||||
|
||||
// jsonColumns are PocketBase JSON fields, which may be NULL in the source and
|
||||
// map to a NOT NULL text column defaulting to the empty string. unwrap marks the fields the legacy reader
|
||||
// decoded from a JSON string (jsonTextToString) rather than returning raw.
|
||||
var jsonColumns = map[string]struct{ unwrap bool }{
|
||||
"captures.headers": {},
|
||||
"mappings.fields": {},
|
||||
"tool_calls.args": {unwrap: true},
|
||||
"tool_calls.result": {unwrap: true},
|
||||
}
|
||||
|
||||
// timeLayouts are every layout a legacy timestamp was written in: the
|
||||
// fixed-width cursor layout, Go's RFC 3339 forms, PocketBase's autodate
|
||||
// layout and SQLite's datetime('now').
|
||||
var timeLayouts = []string{
|
||||
"2006-01-02T15:04:05.000000000Z",
|
||||
time.RFC3339Nano,
|
||||
time.RFC3339,
|
||||
"2006-01-02 15:04:05.999999999Z07:00",
|
||||
"2006-01-02 15:04:05.999999999",
|
||||
}
|
||||
|
||||
type column struct {
|
||||
name string
|
||||
dataType string
|
||||
identity bool
|
||||
source int // index into the legacy row
|
||||
}
|
||||
|
||||
// loadedTable is what load wrote, for verify to check.
|
||||
type loadedTable struct {
|
||||
table legacyread.Table
|
||||
rows legacyread.Rows
|
||||
keys []string // target keys of the rows written, in source read order
|
||||
keyCols []string
|
||||
identity bool
|
||||
}
|
||||
|
||||
// load reads t from src, converts every row to the target's types and copies
|
||||
// them in with their historical ids. A row it cannot convert is added to
|
||||
// problems and nothing of the table is written.
|
||||
func load(ctx context.Context, tx pgx.Tx, src *legacyread.Source, t legacyread.Table, problems *refusal) (loadedTable, error) {
|
||||
cols, err := plan(ctx, tx, t, problems)
|
||||
if err != nil || cols == nil {
|
||||
return loadedTable{table: t}, err
|
||||
}
|
||||
rows, err := src.Read(ctx, t)
|
||||
if err != nil {
|
||||
return loadedTable{}, err
|
||||
}
|
||||
lt := loadedTable{table: t, rows: rows, keyCols: t.Key}
|
||||
if k, ok := targetKeys[t.Name]; ok {
|
||||
lt.keyCols = k
|
||||
}
|
||||
names := make([]string, len(cols))
|
||||
for i, c := range cols {
|
||||
names[i] = c.name
|
||||
lt.identity = lt.identity || c.identity
|
||||
}
|
||||
out, ok := convertRows(lt, cols, names, problems)
|
||||
if !ok {
|
||||
return lt, nil
|
||||
}
|
||||
for _, row := range out {
|
||||
key := make([]string, len(lt.keyCols))
|
||||
for j, k := range lt.keyCols {
|
||||
key[j] = fmt.Sprint(row[slices.Index(names, k)])
|
||||
}
|
||||
lt.keys = append(lt.keys, strings.Join(key, "/"))
|
||||
}
|
||||
if len(out) > 0 {
|
||||
if _, err := tx.CopyFrom(ctx, pgx.Identifier{t.Name}, names, pgx.CopyFromRows(out)); err != nil {
|
||||
problems.add("%s: the target rejected the rows: %v", t.Name, err)
|
||||
return lt, nil
|
||||
}
|
||||
}
|
||||
if lt.identity {
|
||||
return lt, advanceSequence(ctx, tx, src, t.Name)
|
||||
}
|
||||
return lt, nil
|
||||
}
|
||||
|
||||
// convertRows converts every row of lt, adding each value it cannot convert
|
||||
// to problems.
|
||||
func convertRows(lt loadedTable, cols []column, names []string, problems *refusal) ([][]any, bool) {
|
||||
sourceKeys := lt.rows.Keys()
|
||||
out := make([][]any, 0, len(lt.rows.Rows))
|
||||
ok := true
|
||||
for i, row := range lt.rows.Rows {
|
||||
converted := make([]any, len(cols))
|
||||
for j, c := range cols {
|
||||
v, err := convert(lt.table.Name+"."+c.name, c.dataType, row[c.source])
|
||||
if err != nil {
|
||||
problems.add("%s %s: column %s value %s: %v", lt.table.Name, sourceKeys[i], names[j], quote(row[c.source]), err)
|
||||
ok = false
|
||||
}
|
||||
converted[j] = v
|
||||
}
|
||||
out = append(out, converted)
|
||||
}
|
||||
return out, ok
|
||||
}
|
||||
|
||||
// advanceSequence sets table's identity sequence to hand out the id the
|
||||
// legacy AUTOINCREMENT would have, even for a table whose rows were all
|
||||
// deleted. is_called = false makes that id the next one, which keeps
|
||||
// AUTOINCREMENT's promise that a deleted top id is never reused.
|
||||
func advanceSequence(ctx context.Context, tx pgx.Tx, src *legacyread.Source, table string) error {
|
||||
seq, err := src.SequenceOf(ctx, table)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "SELECT setval(pg_get_serial_sequence($1, 'id'), $2, false)", table, seq.Next); err != nil {
|
||||
return fmt.Errorf("legacy import: advance %s sequence: %w", table, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// plan matches the target table's columns to the legacy ones. A column on
|
||||
// either side with no counterpart is a schema gap the import refuses to
|
||||
// guess about; it returns nil columns then.
|
||||
func plan(ctx context.Context, tx pgx.Tx, t legacyread.Table, problems *refusal) ([]column, error) {
|
||||
rows, err := tx.Query(ctx, `SELECT column_name, data_type, is_identity = 'YES'
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = $1 AND is_generated = 'NEVER'
|
||||
ORDER BY ordinal_position`, t.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacy import: %s columns: %w", t.Name, err)
|
||||
}
|
||||
cols, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (column, error) {
|
||||
var c column
|
||||
err := row.Scan(&c.name, &c.dataType, &c.identity)
|
||||
return c, err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacy import: %s columns: %w", t.Name, err)
|
||||
}
|
||||
gap := len(cols) == 0
|
||||
if gap {
|
||||
problems.add("%s: no target table", t.Name)
|
||||
}
|
||||
targetNames := map[string]bool{}
|
||||
for i := range cols {
|
||||
targetNames[cols[i].name] = true
|
||||
cols[i].source = slices.Index(t.Columns, cols[i].name)
|
||||
if cols[i].source < 0 {
|
||||
problems.add("%s.%s: the target column has no legacy column", t.Name, cols[i].name)
|
||||
gap = true
|
||||
}
|
||||
}
|
||||
for _, c := range t.Columns {
|
||||
if !targetNames[c] && !droppedColumns[t.Name+"."+c] {
|
||||
problems.add("%s.%s: the legacy column has no target column", t.Name, c)
|
||||
gap = true
|
||||
}
|
||||
}
|
||||
if gap {
|
||||
return nil, nil
|
||||
}
|
||||
return cols, nil
|
||||
}
|
||||
|
||||
// convert maps one SQLite value to the Go value pgx writes to a column of
|
||||
// dataType. It never coerces: a value the column cannot hold is an error
|
||||
// naming the rule it breaks.
|
||||
func convert(qualified, dataType string, v any) (any, error) {
|
||||
if j, ok := jsonColumns[qualified]; ok {
|
||||
return convertJSON(v, j.unwrap)
|
||||
}
|
||||
if v == nil {
|
||||
return nil, fmt.Errorf("NULL in a NOT NULL %s column", dataType)
|
||||
}
|
||||
conv, ok := converters[dataType]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unsupported target type %s", dataType)
|
||||
}
|
||||
return conv(v)
|
||||
}
|
||||
|
||||
func convertJSON(v any, unwrap bool) (any, error) {
|
||||
switch s := v.(type) {
|
||||
case nil:
|
||||
return "", nil
|
||||
case string:
|
||||
if unwrap {
|
||||
var decoded string
|
||||
if json.Unmarshal([]byte(s), &decoded) == nil {
|
||||
return decoded, nil
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
return nil, errors.New("text: not a string")
|
||||
}
|
||||
|
||||
// converters are keyed by information_schema data_type.
|
||||
var converters = map[string]func(any) (any, error){
|
||||
"text": func(v any) (any, error) {
|
||||
if s, ok := v.(string); ok {
|
||||
return s, nil
|
||||
}
|
||||
return nil, errors.New("text: not a string")
|
||||
},
|
||||
"bigint": func(v any) (any, error) {
|
||||
switch n := v.(type) {
|
||||
case int64:
|
||||
return n, nil
|
||||
case float64:
|
||||
if n == math.Trunc(n) && math.Abs(n) < 1<<63 {
|
||||
return int64(n), nil
|
||||
}
|
||||
}
|
||||
return nil, errors.New("bigint: not an integer")
|
||||
},
|
||||
"boolean": func(v any) (any, error) {
|
||||
switch b := v.(type) {
|
||||
case bool:
|
||||
return b, nil
|
||||
case int64:
|
||||
if b == 0 || b == 1 {
|
||||
return b == 1, nil
|
||||
}
|
||||
}
|
||||
return nil, errors.New("boolean: not 0 or 1")
|
||||
},
|
||||
"bytea": func(v any) (any, error) {
|
||||
switch b := v.(type) {
|
||||
case []byte:
|
||||
return b, nil
|
||||
case string:
|
||||
return []byte(b), nil
|
||||
}
|
||||
return nil, errors.New("bytea: not a blob")
|
||||
},
|
||||
"timestamp with time zone": func(v any) (any, error) {
|
||||
if s, ok := v.(string); ok {
|
||||
for _, layout := range timeLayouts {
|
||||
if at, err := time.Parse(layout, s); err == nil {
|
||||
return at.UTC(), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, errors.New("timestamptz: not a timestamp in any legacy layout")
|
||||
},
|
||||
}
|
||||
|
||||
func quote(v any) string {
|
||||
switch s := v.(type) {
|
||||
case nil:
|
||||
return "NULL"
|
||||
case string:
|
||||
return fmt.Sprintf("%q", s)
|
||||
case []byte:
|
||||
return fmt.Sprintf("x'%x'", s)
|
||||
}
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package legacyimport
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/legacyread"
|
||||
)
|
||||
|
||||
// blobColumns are the columns whose bytes must survive exactly: resource
|
||||
// documents, and binding secrets, which stay decryptable by the configured
|
||||
// keyring only if the envelope is untouched.
|
||||
var blobColumns = map[string]string{
|
||||
"resources": "value",
|
||||
"resource_history": "value",
|
||||
"bindings": "secret",
|
||||
}
|
||||
|
||||
// verify reads back what load wrote for lt and compares it with the source:
|
||||
// the key set (a count passes on a swallowed row when ids have gaps), the
|
||||
// (at, id) event order, blob bytes and the identity sequence.
|
||||
func verify(ctx context.Context, tx pgx.Tx, lt loadedTable, problems *refusal) error {
|
||||
name := lt.table.Name
|
||||
ident := pgx.Identifier{name}.Sanitize()
|
||||
keyExprs := make([]string, len(lt.keyCols))
|
||||
for i, k := range lt.keyCols {
|
||||
keyExprs[i] = pgx.Identifier{k}.Sanitize() + "::text"
|
||||
}
|
||||
keyExpr := "concat_ws('/', " + strings.Join(keyExprs, ", ") + ")"
|
||||
|
||||
targetKeys, err := strs(ctx, tx, "SELECT "+keyExpr+" FROM "+ident)
|
||||
if err != nil {
|
||||
return fmt.Errorf("legacy import: verify %s: %w", name, err)
|
||||
}
|
||||
if len(targetKeys) != len(lt.keys) {
|
||||
problems.add("%s: %d rows read, %d rows in the target", name, len(lt.keys), len(targetKeys))
|
||||
}
|
||||
if d := legacyread.CompareKeys(lt.keys, targetKeys); len(d.Missing)+len(d.Extra) > 0 {
|
||||
problems.add("%s: rows missing from the target %v, rows only in the target %v", name, d.Missing, d.Extra)
|
||||
}
|
||||
|
||||
if name == "events" {
|
||||
if err := verifyEventOrder(ctx, tx, lt, problems); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if col, ok := blobColumns[name]; ok {
|
||||
if err := verifyBlobs(ctx, tx, lt, col, keyExpr, problems); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if lt.identity {
|
||||
return verifySequence(ctx, tx, name, problems)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifySequence asserts the next id the sequence hands out is above every
|
||||
// imported id, reading its state rather than calling nextval.
|
||||
func verifySequence(ctx context.Context, tx pgx.Tx, name string, problems *refusal) error {
|
||||
var seq string
|
||||
var maxID int64
|
||||
if err := tx.QueryRow(ctx, "SELECT pg_get_serial_sequence($1, 'id'), (SELECT COALESCE(max(id), 0) FROM "+
|
||||
pgx.Identifier{name}.Sanitize()+")", name).Scan(&seq, &maxID); err != nil {
|
||||
return fmt.Errorf("legacy import: verify %s sequence: %w", name, err)
|
||||
}
|
||||
var (
|
||||
last int64
|
||||
called bool
|
||||
)
|
||||
// seq comes from pg_get_serial_sequence, which returns it quoted.
|
||||
if err := tx.QueryRow(ctx, "SELECT last_value, is_called FROM "+seq).Scan(&last, &called); err != nil {
|
||||
return fmt.Errorf("legacy import: verify %s sequence: %w", name, err)
|
||||
}
|
||||
next := last
|
||||
if called {
|
||||
next++
|
||||
}
|
||||
if next <= maxID {
|
||||
problems.add("%s: next id %d would reuse an imported id (max %d)", name, next, maxID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyEventOrder compares the ids in (at, id) order on both sides. The
|
||||
// source was read in that order; a difference means microsecond storage
|
||||
// reordered history, and a cursor reader would skip a row.
|
||||
func verifyEventOrder(ctx context.Context, tx pgx.Tx, lt loadedTable, problems *refusal) error {
|
||||
source := make([]int64, len(lt.keys))
|
||||
for i, k := range lt.keys {
|
||||
id, err := strconv.ParseInt(k, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("legacy import: event key %q: %w", k, err)
|
||||
}
|
||||
source[i] = id
|
||||
}
|
||||
rows, err := tx.Query(ctx, "SELECT id FROM events ORDER BY at, id")
|
||||
if err != nil {
|
||||
return fmt.Errorf("legacy import: verify events order: %w", err)
|
||||
}
|
||||
target, err := pgx.CollectRows(rows, pgx.RowTo[int64])
|
||||
if err != nil {
|
||||
return fmt.Errorf("legacy import: verify events order: %w", err)
|
||||
}
|
||||
if d := legacyread.CompareOrder(source, target); d != nil {
|
||||
problems.add("events: (at, id) order differs at position %d: source id %d, target id %d", d.Position, d.Source, d.Target)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyBlobs(ctx context.Context, tx pgx.Tx, lt loadedTable, col, keyExpr string, problems *refusal) error {
|
||||
source := map[string][]byte{}
|
||||
for i, k := range lt.keys {
|
||||
switch v := lt.rows.Value(i, col).(type) {
|
||||
case []byte:
|
||||
source[k] = v
|
||||
case string:
|
||||
source[k] = []byte(v)
|
||||
}
|
||||
}
|
||||
rows, err := tx.Query(ctx, fmt.Sprintf("SELECT %s, %s FROM %s", keyExpr,
|
||||
pgx.Identifier{col}.Sanitize(), pgx.Identifier{lt.table.Name}.Sanitize()))
|
||||
if err != nil {
|
||||
return fmt.Errorf("legacy import: verify %s.%s: %w", lt.table.Name, col, err)
|
||||
}
|
||||
target := map[string][]byte{}
|
||||
for rows.Next() {
|
||||
var (
|
||||
k string
|
||||
v []byte
|
||||
)
|
||||
if err := rows.Scan(&k, &v); err != nil {
|
||||
rows.Close()
|
||||
return fmt.Errorf("legacy import: verify %s.%s: %w", lt.table.Name, col, err)
|
||||
}
|
||||
target[k] = v
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("legacy import: verify %s.%s: %w", lt.table.Name, col, err)
|
||||
}
|
||||
if diff := legacyread.CompareBlobs(source, target); len(diff) > 0 {
|
||||
problems.add("%s.%s: bytes differ from the source for %v", lt.table.Name, col, diff)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unindexedMessages returns the source message ids the FTS5 index missed.
|
||||
func unindexedMessages(ctx context.Context, gw *legacyread.Source, loaded []loadedTable) ([]string, error) {
|
||||
indexed, err := gw.IndexedMessageIDs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, lt := range loaded {
|
||||
if lt.table.Name == "messages" {
|
||||
return legacyread.CompareKeys(indexed, lt.keys).Extra, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func strs(ctx context.Context, tx pgx.Tx, query string) ([]string, error) {
|
||||
rows, err := tx.Query(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package legacyread
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// HasData reports whether the legacy file at path holds any row in tables. A
|
||||
// missing file, and a table the file never created, hold nothing. It is the
|
||||
// legacy half of the boot gate: a Postgres database without a completed import
|
||||
// must not serve while this is true for any source.
|
||||
func HasData(ctx context.Context, path string, tables []Table) (bool, error) {
|
||||
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
src, err := Open(ctx, path)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = src.Close() }()
|
||||
for _, t := range tables {
|
||||
var present bool
|
||||
if err := src.db.QueryRowContext(ctx,
|
||||
"SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)", t.Name).Scan(&present); err != nil {
|
||||
return false, fmt.Errorf("legacyread: %s: %w", path, err)
|
||||
}
|
||||
if !present {
|
||||
continue
|
||||
}
|
||||
var rows bool
|
||||
if err := src.db.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM "+t.Name+")").Scan(&rows); err != nil {
|
||||
return false, fmt.Errorf("legacyread: %s: %s: %w", path, t.Name, err)
|
||||
}
|
||||
if rows {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// SequenceOf returns the state of table's AUTOINCREMENT id column.
|
||||
func (s *Source) SequenceOf(ctx context.Context, table string) (Sequence, error) {
|
||||
seq := Sequence{Table: table}
|
||||
var high int64
|
||||
err := s.db.QueryRowContext(ctx, fmt.Sprintf(
|
||||
"SELECT COALESCE((SELECT max(id) FROM %[1]s), 0), COALESCE((SELECT seq FROM sqlite_sequence WHERE name = '%[1]s'), 0)",
|
||||
table)).Scan(&seq.MaxID, &high)
|
||||
if err != nil {
|
||||
return Sequence{}, fmt.Errorf("legacyread: sequence %s: %w", table, err)
|
||||
}
|
||||
seq.Next = max(seq.MaxID, high) + 1
|
||||
return seq, nil
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package legacyread
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHasData(t *testing.T) {
|
||||
emptyGateway := `CREATE TABLE sessions (session_id TEXT PRIMARY KEY);
|
||||
CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT);`
|
||||
tests := []struct {
|
||||
name string
|
||||
path func(t *testing.T) string
|
||||
tables []Table
|
||||
want bool
|
||||
}{
|
||||
{"missing file", func(t *testing.T) string { return filepath.Join(t.TempDir(), "absent.sqlite") }, GatewayTables, false},
|
||||
{"tables without rows", func(t *testing.T) string { return fixture(t, "gw.sqlite", emptyGateway) }, GatewayTables, false},
|
||||
{"rows present", func(t *testing.T) string { return fixture(t, "gw.sqlite", gatewayDDL) }, GatewayTables, true},
|
||||
{"rows in the state store", func(t *testing.T) string { return fixture(t, "archie.db", stateStoreDDL) }, StateStoreTables, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := HasData(t.Context(), tt.path(t), tt.tables)
|
||||
if err != nil {
|
||||
t.Fatalf("HasData: %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("HasData = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSequenceOfMessages(t *testing.T) {
|
||||
path := fixture(t, "gw.sqlite", gatewayDDL)
|
||||
exec(t, path, `DELETE FROM messages WHERE message_id = 'm2'`)
|
||||
src := openSource(t, path)
|
||||
got, err := src.SequenceOf(t.Context(), "messages")
|
||||
if err != nil {
|
||||
t.Fatalf("SequenceOf: %v", err)
|
||||
}
|
||||
if want := (Sequence{Table: "messages", MaxID: 1, Next: 3}); got != want {
|
||||
t.Fatalf("SequenceOf(messages) = %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -242,15 +242,10 @@ var sequenceTables = []string{"tasks", "transitions", "events", "resource_histor
|
||||
func (s *Source) Sequences(ctx context.Context) ([]Sequence, error) {
|
||||
out := make([]Sequence, 0, len(sequenceTables))
|
||||
for _, table := range sequenceTables {
|
||||
seq := Sequence{Table: table}
|
||||
var high int64
|
||||
err := s.db.QueryRowContext(ctx, fmt.Sprintf(
|
||||
"SELECT COALESCE((SELECT max(id) FROM %[1]s), 0), COALESCE((SELECT seq FROM sqlite_sequence WHERE name = '%[1]s'), 0)",
|
||||
table)).Scan(&seq.MaxID, &high)
|
||||
seq, err := s.SequenceOf(ctx, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacyread: sequence %s: %w", table, err)
|
||||
return nil, err
|
||||
}
|
||||
seq.Next = max(seq.MaxID, high) + 1
|
||||
out = append(out, seq)
|
||||
}
|
||||
return out, nil
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb"
|
||||
)
|
||||
|
||||
// Service owner names: every Archie process that serves this database holds
|
||||
// one of these claims for its whole life, so an offline rewrite checks them
|
||||
// all.
|
||||
const (
|
||||
OwnerStateStore = "state-store"
|
||||
OwnerGateway = "gateway"
|
||||
)
|
||||
|
||||
// ImportComplete reports whether the one-time legacy import committed and
|
||||
// verified against this migrated database. Serving over non-empty legacy
|
||||
// files is refused without it.
|
||||
func ImportComplete(ctx context.Context, pool *pgxpool.Pool) (bool, error) {
|
||||
done, err := postgresdb.New(pool).ImportCompleted(ctx)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("postgres: import completion: %w", err)
|
||||
}
|
||||
return done, nil
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
|
||||
)
|
||||
|
||||
func TestImportComplete(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
record bool
|
||||
want bool
|
||||
}{
|
||||
{name: "migrated database without a record", want: false},
|
||||
{name: "committed record", record: true, want: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
pool := openPool(t, pgtest.URL(t))
|
||||
if err := Migrate(t.Context(), pool, Migrations()); err != nil {
|
||||
t.Fatalf("Migrate: %v", err)
|
||||
}
|
||||
if tt.record {
|
||||
if _, err := pool.Exec(t.Context(),
|
||||
"INSERT INTO import_completion (id, sources, report) VALUES (1, 's', 'r')"); err != nil {
|
||||
t.Fatalf("seed record: %v", err)
|
||||
}
|
||||
}
|
||||
got, err := ImportComplete(t.Context(), pool)
|
||||
if err != nil {
|
||||
t.Fatalf("ImportComplete: %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("ImportComplete = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
-- import_completion records that the one-time import of an existing install's
|
||||
-- SQLite stores committed and passed verification. It is written in the same
|
||||
-- transaction as the imported rows, so a record exists exactly when the data
|
||||
-- does. An empty database is not evidence that an import succeeded, only that
|
||||
-- nothing has written to it yet, so serving over non-empty legacy files
|
||||
-- requires this row.
|
||||
|
||||
-- +goose Up
|
||||
|
||||
CREATE TABLE import_completion (
|
||||
id bigint PRIMARY KEY CHECK (id = 1),
|
||||
completed_at timestamptz NOT NULL DEFAULT now(),
|
||||
sources text NOT NULL,
|
||||
report text NOT NULL
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
|
||||
DROP TABLE import_completion;
|
||||
@@ -0,0 +1,37 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: import.sql
|
||||
|
||||
package postgresdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
const importCompleted = `-- name: ImportCompleted :one
|
||||
SELECT EXISTS (SELECT 1 FROM import_completion)
|
||||
`
|
||||
|
||||
func (q *Queries) ImportCompleted(ctx context.Context) (bool, error) {
|
||||
row := q.db.QueryRow(ctx, importCompleted)
|
||||
var exists bool
|
||||
err := row.Scan(&exists)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
const insertImportCompletion = `-- name: InsertImportCompletion :exec
|
||||
|
||||
INSERT INTO import_completion (id, sources, report) VALUES (1, $1, $2)
|
||||
`
|
||||
|
||||
type InsertImportCompletionParams struct {
|
||||
Sources string
|
||||
Report string
|
||||
}
|
||||
|
||||
// The one-time legacy import's completion record.
|
||||
func (q *Queries) InsertImportCompletion(ctx context.Context, arg InsertImportCompletionParams) error {
|
||||
_, err := q.db.Exec(ctx, insertImportCompletion, arg.Sources, arg.Report)
|
||||
return err
|
||||
}
|
||||
@@ -119,6 +119,13 @@ type IdentitySubject struct {
|
||||
BoundAt time.Time
|
||||
}
|
||||
|
||||
type ImportCompletion struct {
|
||||
ID int64
|
||||
CompletedAt time.Time
|
||||
Sources string
|
||||
Report string
|
||||
}
|
||||
|
||||
type Mapping struct {
|
||||
ID string
|
||||
Name string
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
-- The one-time legacy import's completion record.
|
||||
|
||||
-- name: InsertImportCompletion :exec
|
||||
INSERT INTO import_completion (id, sources, report) VALUES (1, $1, $2);
|
||||
|
||||
-- name: ImportCompleted :one
|
||||
SELECT EXISTS (SELECT 1 FROM import_completion);
|
||||
Reference in new issue
Block a user