From 42dcc8075468a7f016fffb7d46f057a596d321f1 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Thu, 24 Sep 2026 09:45:26 +1000 Subject: [PATCH] feat(state-store): one-time import of the legacy SQLite stores into Postgres archie-state-store import copies the State Store task database, the PocketBase event store and the Gateway conversation store, all named from db_path, into Postgres in one transaction. The bead asked for an archied command; offline maintenance of a store belongs to the binary that owns it, so it sits beside backup/restore/validate/rollback instead. It refuses rather than guesses: while the state-store or gateway ownership claim is held, when a domain it imports already holds rows (an allowlist of that domain's tables, checked after Migrate), on a completed prior import, on FK orphans, alias case collisions, duplicate bindings, events whose (at, id) order microsecond storage would change, schema gaps, and any value the target column cannot hold, naming table, row, value and rule. Historical ids are kept and every identity sequence is set from AUTOINCREMENT's high-water mark. After loading it compares key sets, event order, resource blobs and binding secrets byte for byte and the next id of every sequence, checks the source files were not written while read, and only then writes an import_completion row (migration 0005) in the same transaction. postgres.ImportComplete and legacyimport.HasLegacyData are the two halves of the boot gate: legacy data without a completed import must not serve. --- cmd/archie-state-store/import_test.go | 49 ++ cmd/archie-state-store/main.go | 2 +- cmd/archie-state-store/recovery.go | 17 +- cmd/archie-state-store/recovery_test.go | 2 +- internal/app/archied/state_store_import.go | 56 ++ internal/app/archied/state_store_recovery.go | 27 +- .../legacyimport/legacyimport.go | 387 +++++++++++++ .../legacyimport/legacyimport_test.go | 517 ++++++++++++++++++ internal/infrastructure/legacyimport/load.go | 288 ++++++++++ .../infrastructure/legacyimport/verify.go | 171 ++++++ .../infrastructure/legacyread/presence.go | 55 ++ .../legacyread/presence_test.go | 46 ++ internal/infrastructure/legacyread/verify.go | 9 +- .../postgres/import_completion.go | 29 + .../postgres/import_completion_test.go | 39 ++ .../migrations/0005_import_completion.sql | 19 + .../postgres/postgresdb/import.sql.go | 37 ++ .../postgres/postgresdb/models.go | 7 + .../postgres/queries/import.sql | 7 + 19 files changed, 1743 insertions(+), 21 deletions(-) create mode 100644 cmd/archie-state-store/import_test.go create mode 100644 internal/app/archied/state_store_import.go create mode 100644 internal/infrastructure/legacyimport/legacyimport.go create mode 100644 internal/infrastructure/legacyimport/legacyimport_test.go create mode 100644 internal/infrastructure/legacyimport/load.go create mode 100644 internal/infrastructure/legacyimport/verify.go create mode 100644 internal/infrastructure/legacyread/presence.go create mode 100644 internal/infrastructure/legacyread/presence_test.go create mode 100644 internal/infrastructure/postgres/import_completion.go create mode 100644 internal/infrastructure/postgres/import_completion_test.go create mode 100644 internal/infrastructure/postgres/migrations/0005_import_completion.sql create mode 100644 internal/infrastructure/postgres/postgresdb/import.sql.go create mode 100644 internal/infrastructure/postgres/queries/import.sql diff --git a/cmd/archie-state-store/import_test.go b/cmd/archie-state-store/import_test.go new file mode 100644 index 000000000..f0a61458c --- /dev/null +++ b/cmd/archie-state-store/import_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/infrastructure/postgres" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" +) + +func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) } + +// TestRecoveryImportMovesTheInstallOnce drives the import the way an operator +// does: the configured db_path locates every legacy file, and a second run +// against the same database is refused because the first one completed. +func TestRecoveryImportMovesTheInstallOnce(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "archie.db") + st := openTaskStore(t, dbPath+"-tasks.sqlite") + seedTask(t, st, "carried over") + if err := st.Close(); err != nil { + t.Fatalf("close task store: %v", err) + } + url := pgtest.URL(t) + + code, stdout, stderr := runRecoveryCmd(t, "import", "-db-path", dbPath, "-database-url", url) + if code != 0 { + t.Fatalf("import exited %d; stderr = %q", code, stderr) + } + if !strings.Contains(stdout, "tasks=1") { + t.Errorf("import summary %q does not report the imported task", stdout) + } + + pool, err := postgres.Open(t.Context(), url) + if err != nil { + t.Fatalf("postgres.Open: %v", err) + } + defer pool.Close() + var title string + if err := pool.QueryRow(t.Context(), "SELECT title FROM tasks").Scan(&title); err != nil || title != "carried over" { + t.Fatalf("imported task title = %q, %v; want %q", title, err, "carried over") + } + + code, _, stderr = runRecoveryCmd(t, "import", "-db-path", dbPath, "-database-url", url) + if code != 1 || !strings.Contains(stderr, "already holds a completed import") { + t.Fatalf("second import exited %d, stderr %q; want 1 naming the completed import", code, stderr) + } +} diff --git a/cmd/archie-state-store/main.go b/cmd/archie-state-store/main.go index c664fa746..c95b8f2c2 100644 --- a/cmd/archie-state-store/main.go +++ b/cmd/archie-state-store/main.go @@ -40,7 +40,7 @@ func run() int { flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)") flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)") flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store -h\n\n") + fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store -h\n\n") flag.PrintDefaults() } flag.Parse() diff --git a/cmd/archie-state-store/recovery.go b/cmd/archie-state-store/recovery.go index 4d115a338..d96cfe466 100644 --- a/cmd/archie-state-store/recovery.go +++ b/cmd/archie-state-store/recovery.go @@ -31,6 +31,9 @@ owns it, while backup takes a consistent snapshot of a serving store. configuration the daemon would boot with rollback -db FILE -kind KIND replay an earlier revision of a stored [-revision N] resource through the ordinary replace + import [-db-path PATH] copy the legacy SQLite stores named from + [-database-url URL] db_path into an empty Postgres database, + [-config FILE] once; refuses while any Archie service runs ` // runRecovery parses and performs one offline recovery command, reporting the @@ -44,7 +47,8 @@ func runRecovery(args []string, stdout, stderr io.Writer) int { command := args[0] options := archied.StateStoreRecoveryOptions{Operation: command} switch command { - case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback: + case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback, + archied.RecoveryImport: default: fmt.Fprintf(stderr, "archie-state-store: unknown command %q\n\n", command) fmt.Fprint(stderr, recoveryUsage) @@ -59,9 +63,14 @@ func runRecovery(args []string, stdout, stderr io.Writer) int { flags.StringVar(&options.From, "from", "", "snapshot file restore reads") flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays") flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)") - // Only validate asks a question about the process rather than the file, so - // only validate takes the configuration the process would boot with. - if command == archied.RecoveryValidate { + // validate asks a question about the process rather than the file, and + // import reads db_path and database_url from it, so only those two take + // the configuration the process would boot with. + if command == archied.RecoveryImport { + flags.StringVar(&options.DBPath, "db-path", "", "configured db_path the legacy files are named from (default: the configuration's)") + flags.StringVar(&options.DatabaseURL, "database-url", "", "Postgres URL to import into (default: the configuration's database_url)") + } + if command == archied.RecoveryValidate || command == archied.RecoveryImport { flags.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory the daemon boots with") flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") } diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go index 0ab53025c..436e18306 100644 --- a/cmd/archie-state-store/recovery_test.go +++ b/cmd/archie-state-store/recovery_test.go @@ -356,7 +356,7 @@ func TestRecoveryValidateRunsTheGateBootRuns(t *testing.T) { // path in the same binary exits 0 for the same request, so a script probing the // recovery surface must not read a requested help as a failure. func TestRecoveryHelpExitsZero(t *testing.T) { - commands := []string{archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback} + commands := []string{archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback, archied.RecoveryImport} for _, command := range commands { code, _, stderr := runRecoveryCmd(t, command, "-h") if code != 0 { diff --git a/internal/app/archied/state_store_import.go b/internal/app/archied/state_store_import.go new file mode 100644 index 000000000..dc7f165be --- /dev/null +++ b/internal/app/archied/state_store_import.go @@ -0,0 +1,56 @@ +package archied + +import ( + "context" + "errors" + "fmt" + + "github.com/samcharles93/archie-core/internal/infrastructure/legacyimport" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres" +) + +// legacySources names an install's three legacy SQLite files from its +// configured db_path, the same derivation the legacy openers use, so an +// operator cannot import one store and forget another. +func legacySources(dbPath string) legacyimport.Sources { + return legacyimport.Sources{ + StateStore: taskDBPath(dbPath), + EDA: edaDBPath(dbPath), + Gateway: conversationDBPath(dbPath), + } +} + +// importLegacy runs the one-time import of the legacy files into Postgres. +// It is an offline command of the State Store binary, not of archied: offline +// maintenance of a store belongs to the process that owns it. +func importLegacy(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + dbPath, url := options.DBPath, options.DatabaseURL + if dbPath == "" || url == "" { + cfg, err := bootConfig(ctx, options) + if err != nil { + return "", err + } + if dbPath == "" { + dbPath = cfg.DBPath + } + if url == "" { + url = cfg.DatabaseURL + } + } + if dbPath == "" { + return "", errors.New("import requires db_path (-db-path or the configuration)") + } + if url == "" { + return "", errors.New("import requires database_url (-database-url or the configuration)") + } + pool, err := postgres.Open(ctx, url) + if err != nil { + return "", err + } + defer pool.Close() + report, err := legacyimport.Import(ctx, pool, legacySources(dbPath)) + if err != nil { + return "", err + } + return fmt.Sprintf("imported %s into Postgres and recorded the completion: %s", dbPath, report), nil +} diff --git a/internal/app/archied/state_store_recovery.go b/internal/app/archied/state_store_recovery.go index fe4636726..04ee111c5 100644 --- a/internal/app/archied/state_store_recovery.go +++ b/internal/app/archied/state_store_recovery.go @@ -46,6 +46,7 @@ const ( RecoveryRestore = "restore" RecoveryValidate = "validate" RecoveryRollback = "rollback" + RecoveryImport = "import" ) // StateStoreRecoveryOptions are the process inputs for one offline recovery @@ -71,6 +72,11 @@ type StateStoreRecoveryOptions struct { // Revision is the revision rollback replays. Zero means the newest // revision older than the one the resource carries now. Revision int64 + // DBPath and DatabaseURL are import's source and target: the configured + // db_path every legacy file is named from, and the Postgres URL. Either + // left empty is read from the configuration. + DBPath string + DatabaseURL string } // RunStateStoreRecovery performs one offline operation on the task database and @@ -81,18 +87,13 @@ func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOption if options.DB == "" || options.Out == "" { return "", errors.New("backup requires -db and -out") } - if err := store.Backup(ctx, options.DB, options.Out); err != nil { - return "", err - } - return fmt.Sprintf("backed up %s to %s", options.DB, options.Out), nil + return summarise(store.Backup(ctx, options.DB, options.Out), "backed up %s to %s", options.DB, options.Out) case RecoveryRestore: if options.DB == "" || options.From == "" { return "", errors.New("restore requires -db and -from") } - if err := store.Restore(ctx, options.DB, options.From); err != nil { - return "", err - } - return fmt.Sprintf("restored %s from %s; start the State Store again", options.DB, options.From), nil + return summarise(store.Restore(ctx, options.DB, options.From), + "restored %s from %s; start the State Store again", options.DB, options.From) case RecoveryValidate: if options.DB == "" { return "", errors.New("validate requires -db") @@ -103,11 +104,21 @@ func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOption return "", errors.New("rollback requires -db and -kind") } return rollbackResource(ctx, options) + case RecoveryImport: + return importLegacy(ctx, options) default: return "", fmt.Errorf("unknown recovery command %q", options.Operation) } } +// summarise is an operation's stdout line, or its error. +func summarise(err error, format string, args ...any) (string, error) { + if err != nil { + return "", err + } + return fmt.Sprintf(format, args...), nil +} + // validateStore answers the question an operator actually has: would archied // start against this file. It runs the checks that stop it, in the order boot // meets them. diff --git a/internal/infrastructure/legacyimport/legacyimport.go b/internal/infrastructure/legacyimport/legacyimport.go new file mode 100644 index 000000000..49e5c6e7f --- /dev/null +++ b/internal/infrastructure/legacyimport/legacyimport.go @@ -0,0 +1,387 @@ +// Package legacyimport moves an existing install's SQLite stores (the State +// Store's task database, the PocketBase event store and the Gateway +// conversation store) into Postgres, once. +// +// The import is one transaction: it refuses a target that already holds data +// in a domain it imports, refuses rather than skips a row the target cannot +// hold, verifies what it wrote against what it read, and only then records a +// completion row and commits. A refusal at any point leaves the target as it +// was, migrated and empty, so a failed import is never serveable. +// +// It reads through legacyread, which never writes to the source files, and is +// deleted together with it. +package legacyimport + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "maps" + "os" + "slices" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/samcharles93/archie-core/internal/infrastructure/legacyread" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb" +) + +// ErrRefused wraps every refusal. The message names each offending table, +// row, value and rule. +var ErrRefused = errors.New("legacy import refused") + +// Sources are the legacy files. A path whose file does not exist means that +// domain has nothing to import. +type Sources struct { + StateStore string + EDA string + Gateway string +} + +type domain struct { + name string + path string + tables []legacyread.Table +} + +func (s Sources) domains() []domain { + return []domain{ + {"state-store", s.StateStore, legacyread.StateStoreTables}, + {"eda", s.EDA, legacyread.EDATables}, + {"gateway", s.Gateway, legacyread.GatewayTables}, + } +} + +// present returns the domains whose source file exists. +func (s Sources) present() ([]domain, error) { + var out []domain + for _, d := range s.domains() { + if d.path == "" { + continue + } + _, err := os.Stat(d.path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return nil, fmt.Errorf("legacy import: %w", err) + } + out = append(out, d) + } + return out, nil +} + +// HasLegacyData reports whether any legacy source holds a row. With +// postgres.ImportComplete it is what decides whether a Postgres database may +// serve: legacy data without a completed import must be refused. +func HasLegacyData(ctx context.Context, src Sources) (bool, error) { + for _, d := range src.domains() { + if d.path == "" { + continue + } + has, err := legacyread.HasData(ctx, d.path, d.tables) + if err != nil || has { + return has, err + } + } + return false, nil +} + +// Report is what an import wrote. +type Report struct { + // Rows is each imported table's row count. + Rows map[string]int + // UnindexedMessages are Gateway message ids the source's FTS5 index did + // not cover. The target derives its search column from every row, so + // these become searchable; the count says the source index was stale. + UnindexedMessages []string +} + +func (r Report) String() string { + var b strings.Builder + for _, table := range slices.Sorted(maps.Keys(r.Rows)) { + fmt.Fprintf(&b, "%s=%d ", table, r.Rows[table]) + } + fmt.Fprintf(&b, "unindexed_messages=%d", len(r.UnindexedMessages)) + return b.String() +} + +// refusal collects every problem found before giving up, so an operator fixes +// a source in one pass rather than one row per attempt. +type refusal []string + +func (r *refusal) add(format string, args ...any) { *r = append(*r, fmt.Sprintf(format, args...)) } + +func (r *refusal) err() error { + if len(*r) == 0 { + return nil + } + return fmt.Errorf("%w:\n %s", ErrRefused, strings.Join(*r, "\n ")) +} + +// Import runs the one-time import from src into the database behind pool. +// Every service that serves the database must be stopped: the import holds +// each service's ownership claim for its whole run and refuses if any is +// taken. +func Import(ctx context.Context, pool *pgxpool.Pool, src Sources) (Report, error) { + present, err := src.present() + if err != nil { + return Report{}, err + } + if len(present) == 0 { + return Report{}, fmt.Errorf("%w: no legacy source files found (%s, %s, %s)", ErrRefused, src.StateStore, src.EDA, src.Gateway) + } + before, err := hashSources(present) + if err != nil { + return Report{}, err + } + if err := postgres.Migrate(ctx, pool, postgres.Migrations()); err != nil { + return Report{}, err + } + release, err := claimServices(ctx, pool) + defer release() + if err != nil { + return Report{}, err + } + sources, closeSources, err := openSources(ctx, present) + defer closeSources() + if err != nil { + return Report{}, err + } + + var problems refusal + if err := checkSources(ctx, sources, &problems); err != nil { + return Report{}, err + } + if err := problems.err(); err != nil { + return Report{}, err + } + + tx, err := pool.Begin(ctx) + if err != nil { + return Report{}, fmt.Errorf("legacy import: begin: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + report, err := importInto(ctx, tx, present, sources, &problems) + if err != nil { + return Report{}, err + } + if err := checkUnchanged(before, present, &problems); err != nil { + return Report{}, err + } + if err := problems.err(); err != nil { + return Report{}, err + } + if err := complete(ctx, tx, present, report); err != nil { + return Report{}, err + } + return report, nil +} + +// checkUnchanged refuses when a source's bytes moved during the import: a +// writer that was not stopped would leave the target short of its last writes. +func checkUnchanged(before map[string][32]byte, present []domain, problems *refusal) error { + after, err := hashSources(present) + if err != nil { + return err + } + for path, sum := range before { + if after[path] != sum { + problems.add("%s changed while it was read: stop the process writing it and import again", path) + } + } + return nil +} + +// complete writes the completion record and commits, so the record exists +// exactly when the imported rows do. +func complete(ctx context.Context, tx pgx.Tx, present []domain, report Report) error { + paths := make([]string, len(present)) + for i, d := range present { + paths[i] = d.path + } + if err := postgresdb.New(tx).InsertImportCompletion(ctx, postgresdb.InsertImportCompletionParams{ + Sources: strings.Join(paths, "\n"), + Report: report.String(), + }); err != nil { + return fmt.Errorf("legacy import: completion record: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("legacy import: commit: %w", err) + } + return nil +} + +// claimServices takes every service's ownership claim without waiting. The +// returned release is safe to call whether or not the claims were taken. +func claimServices(ctx context.Context, pool *pgxpool.Pool) (func(), error) { + var claims []*postgres.Ownership + release := func() { + for _, c := range claims { + _ = c.Release(context.WithoutCancel(ctx)) + } + } + for _, name := range []string{postgres.OwnerStateStore, postgres.OwnerGateway} { + claim, err := postgres.AcquireOwnership(ctx, pool, name) + if err != nil { + return release, fmt.Errorf("legacy import: stop every Archie service first: %w", err) + } + claims = append(claims, claim) + } + return release, nil +} + +// openSources opens each present source read-only, keyed by domain name. +func openSources(ctx context.Context, present []domain) (map[string]*legacyread.Source, func(), error) { + sources := map[string]*legacyread.Source{} + closeAll := func() { + for _, s := range sources { + _ = s.Close() + } + } + for _, d := range present { + s, err := legacyread.Open(ctx, d.path) + if err != nil { + return nil, closeAll, err + } + sources[d.name] = s + } + return sources, closeAll, nil +} + +// importInto loads and verifies every present domain inside tx, adding to +// problems anything that must stop the commit. +func importInto(ctx context.Context, tx pgx.Tx, present []domain, sources map[string]*legacyread.Source, problems *refusal) (Report, error) { + if err := checkTarget(ctx, tx, present, problems); err != nil { + return Report{}, err + } + if err := problems.err(); err != nil { + return Report{}, err + } + report := Report{Rows: map[string]int{}} + var loaded []loadedTable + for _, d := range present { + for _, t := range d.tables { + lt, err := load(ctx, tx, sources[d.name], t, problems) + if err != nil { + return Report{}, err + } + loaded = append(loaded, lt) + report.Rows[t.Name] = len(lt.keys) + } + } + if err := problems.err(); err != nil { + return Report{}, err + } + for _, lt := range loaded { + if err := verify(ctx, tx, lt, problems); err != nil { + return Report{}, err + } + } + if gw := sources["gateway"]; gw != nil { + var err error + if report.UnindexedMessages, err = unindexedMessages(ctx, gw, loaded); err != nil { + return Report{}, err + } + } + return report, nil +} + +// hashSources fingerprints each source and its WAL, so a writer that was not +// stopped is caught rather than assumed away. +func hashSources(present []domain) (map[string][32]byte, error) { + out := map[string][32]byte{} + for _, d := range present { + for _, path := range []string{d.path, d.path + "-wal"} { + f, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) && path != d.path { + continue + } + if err != nil { + return nil, fmt.Errorf("legacy import: %w", err) + } + h := sha256.New() + _, err = io.Copy(h, f) + _ = f.Close() + if err != nil { + return nil, fmt.Errorf("legacy import: hash %s: %w", path, err) + } + out[path] = [32]byte(h.Sum(nil)) + } + } + return out, nil +} + +// checkSources finds the source rows Postgres would reject or reorder. +func checkSources(ctx context.Context, sources map[string]*legacyread.Source, problems *refusal) error { + if s := sources["state-store"]; s != nil { + for _, fk := range legacyread.IdentityForeignKeys { + orphans, err := s.Orphans(ctx, fk) + if err != nil { + return err + } + for _, o := range orphans { + problems.add("%s %s: %s %q has no row in %s (foreign key)", fk.Table, o.Key, fk.Column, o.Parent, fk.Parent) + } + } + aliases, err := s.DuplicateAliases(ctx) + if err != nil { + return err + } + for _, d := range aliases { + problems.add("identity_aliases %s: aliases collide on unique lower(alias) %q", strings.Join(d.Values, ", "), d.Key) + } + collisions, err := s.MicrosecondCollisions(ctx) + if err != nil { + return err + } + for _, c := range collisions { + problems.add("events %d and %d: at differs only below a microsecond (%s) while ids run the other way, so (at, id) order would change", + c.EarlierID, c.LaterID, c.At.Format("2006-01-02T15:04:05.000000Z")) + } + } + if s := sources["eda"]; s != nil { + dups, err := s.DuplicateBindings(ctx) + if err != nil { + return err + } + for _, d := range dups { + problems.add("bindings %s: more than one binding for source %q (unique source); resolve them and import again", + strings.Join(d.Values, ", "), d.Key) + } + } + return nil +} + +// checkTarget refuses a database that already completed an import, or that +// holds rows in a table owned by a domain being imported. The check is an +// allowlist of those tables, so goose's version table and other domains' +// tables never count. +func checkTarget(ctx context.Context, tx pgx.Tx, present []domain, problems *refusal) error { + done, err := postgresdb.New(tx).ImportCompleted(ctx) + if err != nil { + return fmt.Errorf("legacy import: completion record: %w", err) + } + if done { + problems.add("this database already holds a completed import") + return nil + } + for _, d := range present { + for _, t := range d.tables { + var rows bool + if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM "+pgx.Identifier{t.Name}.Sanitize()+")").Scan(&rows); err != nil { + return fmt.Errorf("legacy import: %s: %w", t.Name, err) + } + if rows { + problems.add("%s is not empty: the %s target already holds data", t.Name, d.name) + } + } + } + return nil +} diff --git a/internal/infrastructure/legacyimport/legacyimport_test.go b/internal/infrastructure/legacyimport/legacyimport_test.go new file mode 100644 index 000000000..0c07fb6cc --- /dev/null +++ b/internal/infrastructure/legacyimport/legacyimport_test.go @@ -0,0 +1,517 @@ +package legacyimport + +import ( + "bytes" + "crypto/sha256" + "database/sql" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + _ "modernc.org/sqlite" + + "github.com/samcharles93/archie-core/internal/infrastructure/postgres" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" +) + +func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) } + +// The fixtures apply the legacy schemas as their production openers leave +// them, columns added by ALTER TABLE included, with values in the layouts the +// production writers used. +const stateStoreDDL = ` +CREATE TABLE tasks (id INTEGER PRIMARY KEY AUTOINCREMENT, owner TEXT NOT NULL, repo TEXT NOT NULL, + issue_number INTEGER NOT NULL, title TEXT DEFAULT '', body TEXT DEFAULT '', labels TEXT DEFAULT '', + status TEXT DEFAULT 'queued', workflow TEXT DEFAULT '', stage TEXT DEFAULT '', branch TEXT DEFAULT '', + plan TEXT DEFAULT '', notes TEXT DEFAULT '', pr_number INTEGER DEFAULT 0, tokens_used INTEGER DEFAULT 0, + iterations INTEGER DEFAULT 0, attempt INTEGER DEFAULT 0, park_reason TEXT DEFAULT '', + watch_comment_id INTEGER DEFAULT 0, park_class TEXT DEFAULT 'needs_human', remediation_rounds INTEGER DEFAULT 0, + retry_count INTEGER DEFAULT 0, source TEXT DEFAULT 'forge', identity TEXT DEFAULT '', binding_id TEXT DEFAULT '', + binding_version INTEGER DEFAULT 0, review_payload TEXT DEFAULT '', workflow_definition_version INTEGER DEFAULT 0, + workflow_definition_digest TEXT DEFAULT '', workflow_definition_yaml TEXT DEFAULT '', + created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now'))); +ALTER TABLE tasks ADD COLUMN review_cursor INTEGER NOT NULL DEFAULT 0; +CREATE TABLE transitions (id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, + at TEXT DEFAULT (datetime('now')), from_status TEXT NOT NULL, to_status TEXT NOT NULL, detail TEXT DEFAULT ''); +CREATE TABLE events (id INTEGER PRIMARY KEY AUTOINCREMENT, at TEXT NOT NULL, kind TEXT NOT NULL, + task_id INTEGER DEFAULT 0, repo TEXT DEFAULT '', issue INTEGER DEFAULT 0, workflow TEXT DEFAULT '', + stage TEXT DEFAULT '', attempt INTEGER DEFAULT 0, actor_id TEXT DEFAULT '', actor_kind TEXT DEFAULT '', + principal_id TEXT DEFAULT '', detail TEXT DEFAULT '', data TEXT DEFAULT '{}'); +CREATE TABLE resources (kind TEXT PRIMARY KEY, value BLOB NOT NULL, version INTEGER NOT NULL, updated_at TEXT NOT NULL); +CREATE TABLE resource_history (id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, value BLOB NOT NULL, + version INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL, request_id TEXT NOT NULL, + expected_version INTEGER NOT NULL, current_version INTEGER NOT NULL, at TEXT NOT NULL); +CREATE TABLE channel_status (id TEXT PRIMARY KEY, name TEXT DEFAULT '', state TEXT DEFAULT '', detail TEXT DEFAULT '', + configured INTEGER NOT NULL DEFAULT 0, reload_supported INTEGER NOT NULL DEFAULT 0, observed_at TEXT NOT NULL); +CREATE TABLE apply_status (process TEXT NOT NULL, kind TEXT NOT NULL, applied_version INTEGER DEFAULT 0, + error TEXT DEFAULT '', reported_at TEXT NOT NULL, PRIMARY KEY (process, kind)); +CREATE TABLE config_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), schema TEXT DEFAULT '', document TEXT DEFAULT '', + published_at TEXT NOT NULL); +CREATE TABLE identities (id TEXT PRIMARY KEY, kind TEXT NOT NULL, display_name TEXT NOT NULL, lifecycle TEXT NOT NULL, + version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL); +CREATE TABLE identity_aliases (alias TEXT PRIMARY KEY COLLATE NOCASE, identity_id TEXT NOT NULL REFERENCES identities(id)); +CREATE TABLE identity_events (id INTEGER PRIMARY KEY AUTOINCREMENT, identity_id TEXT NOT NULL, event_type TEXT NOT NULL, + from_lifecycle TEXT NOT NULL, to_lifecycle TEXT NOT NULL, display_name TEXT NOT NULL, actor_id TEXT NOT NULL, + source TEXT NOT NULL, request_id TEXT NOT NULL UNIQUE, at TEXT NOT NULL); +CREATE TABLE identity_subjects (issuer TEXT NOT NULL, subject TEXT NOT NULL, + identity_id TEXT NOT NULL REFERENCES identities(id), bound_at TEXT NOT NULL, PRIMARY KEY (issuer, subject)); + +INSERT INTO tasks (id, owner, repo, issue_number, review_cursor, created_at, updated_at) VALUES + (1, 'o', 'r', 7, 3, '2026-01-01 00:00:00', '2026-01-01 00:00:01'), + (4, 'o', 'r', 1000000000000000, 0, '2026-01-02 00:00:00', '2026-01-02 00:00:00'), + (9, 'o', 'r', 8, 0, '2026-01-02 00:00:00', '2026-01-02 00:00:00'); +DELETE FROM tasks WHERE id = 9; +INSERT INTO transitions (task_id, at, from_status, to_status) VALUES (1, '2026-01-01 00:00:00', 'queued', 'running'); +INSERT INTO events (id, at, kind, data) VALUES + (1, '2026-01-01T00:00:00.000000100Z', 'a', 'null'), + (2, '2026-01-01T00:00:01.000000000Z', 'b', '{"k":1}'), + (5, '2026-01-01T00:00:00.500000000Z', 'c', '{}'); +INSERT INTO resources VALUES ('config', X'00ff10', 2, '2026-01-01T00:00:00.123456789Z'); +INSERT INTO resource_history (kind, value, version, actor, source, request_id, expected_version, current_version, at) + VALUES ('config', X'00ff10', 2, 'sam', 'ui', 'req-1', 1, 1, '2026-01-01T00:00:00Z'); +INSERT INTO channel_status VALUES ('tg', 'Telegram', 'up', '', 1, 0, '2026-01-01T00:00:00Z'); +INSERT INTO apply_status VALUES ('daemon', 'config', 2, '', '2026-01-01T00:00:00Z'); +INSERT INTO config_snapshot VALUES (1, 's', '{}', '2026-01-01T00:00:00Z'); +INSERT INTO identities VALUES ('id-1', 'agent', 'Archie', 'active', 1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z'); +INSERT INTO identity_aliases VALUES ('archie', 'id-1'); +INSERT INTO identity_events (identity_id, event_type, from_lifecycle, to_lifecycle, display_name, actor_id, source, request_id, at) + VALUES ('id-1', 'created', '', 'active', 'Archie', 'sam', 'boot', 'rq-1', '2026-01-01T00:00:00Z'); +INSERT INTO identity_subjects VALUES ('https://issuer', 'sub-1', 'id-1', '2026-01-01T00:00:00Z'); +` + +// secret is an encrypted binding secret envelope; it must arrive unchanged. +const secret = "arcie-binding:v1:0123456789abcdef:AbC-_dEf012" + +const edaDDL = ` +CREATE TABLE captures (id TEXT PRIMARY KEY NOT NULL, authenticated BOOLEAN DEFAULT FALSE NOT NULL, + body TEXT DEFAULT '' NOT NULL, content_type TEXT DEFAULT '' NOT NULL, headers JSON DEFAULT NULL, + received_at TEXT DEFAULT '' NOT NULL, remote_addr TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL); +CREATE TABLE mappings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, fields JSON DEFAULT NULL, + name TEXT DEFAULT '' NOT NULL, source_hint TEXT DEFAULT '' NOT NULL, updated_at TEXT DEFAULT '' NOT NULL); +CREATE TABLE bindings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, mapping TEXT DEFAULT '' NOT NULL, + name TEXT DEFAULT '' NOT NULL, owner TEXT DEFAULT '' NOT NULL, repo TEXT DEFAULT '' NOT NULL, + secret TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL, status TEXT DEFAULT '' NOT NULL, + updated_at TEXT DEFAULT '' NOT NULL, version NUMERIC DEFAULT 0 NOT NULL, workflow TEXT DEFAULT '' NOT NULL); +CREATE TABLE binding_dispatches (id TEXT PRIMARY KEY NOT NULL, binding TEXT DEFAULT '' NOT NULL, + binding_version NUMERIC DEFAULT 0 NOT NULL, capture TEXT DEFAULT '' NOT NULL, dispatched_at TEXT DEFAULT '' NOT NULL, + task_id NUMERIC DEFAULT 0 NOT NULL); +CREATE UNIQUE INDEX idx_binding_dispatch_once ON binding_dispatches (binding, capture); +CREATE TABLE playbook_dispatches (id TEXT PRIMARY KEY NOT NULL, action_id TEXT DEFAULT '' NOT NULL, + dispatched_at TEXT DEFAULT '' NOT NULL, event_id TEXT DEFAULT '' NOT NULL, playbook_id TEXT DEFAULT '' NOT NULL, + playbook_version TEXT DEFAULT '' NOT NULL); +CREATE TABLE tool_calls (id TEXT PRIMARY KEY NOT NULL, args JSON DEFAULT NULL, attempt NUMERIC DEFAULT 0 NOT NULL, + called_at TEXT DEFAULT '' NOT NULL, duration_ms NUMERIC DEFAULT 0 NOT NULL, error TEXT DEFAULT '' NOT NULL, + result JSON DEFAULT NULL, task_id NUMERIC DEFAULT 0 NOT NULL, tool TEXT DEFAULT '' NOT NULL); + +INSERT INTO captures VALUES ('rcap1', 1, '{}', 'application/json', '{"x":["1"]}', '2026-01-01T00:00:00.000000000Z', '127.0.0.1', 'gh'); +INSERT INTO captures VALUES ('rcap2', 0, '', '', NULL, '2026-01-01T00:00:01.000000000Z', '', 'gh'); +INSERT INTO mappings VALUES ('rmap1', '2026-01-01 00:00:00.000Z', '[]', 'm', 'gh', '2026-01-01 00:00:00.000Z'); +INSERT INTO bindings VALUES ('rbind1', '2026-01-01 00:00:00.000Z', 'rmap1', 'b', 'o', 'r', '` + secret + `', 'gh', 'armed', '2026-01-01 00:00:00.000Z', 1, 'implement'); +INSERT INTO binding_dispatches VALUES ('rdisp1', 'rbind1', 1, 'rcap1', '2026-01-01 00:00:00.000Z', 4); +INSERT INTO playbook_dispatches VALUES ('rpb1', 'act', '2026-01-01 00:00:00.000Z', 'ev', 'pb', 'v1'); +INSERT INTO tool_calls VALUES ('rtool1', NULL, 1, '2026-01-01 00:00:00.000Z', 12, '', '"exit 0"', 4, 'bash'); +` + +const gatewayDDL = ` +CREATE TABLE sessions (session_id TEXT PRIMARY KEY, platform TEXT NOT NULL DEFAULT '', bot_user TEXT NOT NULL DEFAULT '', + channel_id TEXT NOT NULL DEFAULT '', thread_id TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', + parent_session_id TEXT NOT NULL DEFAULT '', branch_name TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL DEFAULT 0, last_active_at INTEGER NOT NULL DEFAULT 0); +CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT NOT NULL, session_id TEXT NOT NULL, + source_id TEXT NOT NULL DEFAULT '', sender TEXT NOT NULL DEFAULT '', text TEXT NOT NULL DEFAULT '', ts INTEGER NOT NULL, + UNIQUE(session_id, message_id)); +ALTER TABLE messages ADD COLUMN role TEXT NOT NULL DEFAULT ''; +ALTER TABLE messages ADD COLUMN sender_id TEXT NOT NULL DEFAULT ''; +CREATE VIRTUAL TABLE messages_fts USING fts5(sender, text, content='messages', content_rowid='id'); +CREATE TRIGGER messages_ai AFTER INSERT ON messages BEGIN + INSERT INTO messages_fts(rowid, sender, text) VALUES (new.id, new.sender, new.text); +END; +CREATE TABLE turns (turn_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, source_id TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL, attempt INTEGER NOT NULL DEFAULT 0, owner_id TEXT NOT NULL DEFAULT '', + input_message_id TEXT NOT NULL DEFAULT '', assistant_message_id TEXT NOT NULL DEFAULT '', + partial_text TEXT NOT NULL DEFAULT '', response_text TEXT NOT NULL DEFAULT '', tool_calls TEXT NOT NULL DEFAULT '[]', + error TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL); + +INSERT INTO sessions VALUES ('s1', 'telegram', 'bot', 'c1', '', 'Chat', '', '', 1000, 2000); +INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m1', 's1', 'sam', 'hello there', 1000, 'user'); +INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m2', 's1', 'archie', 'hi', 1001, 'assistant'); +INSERT INTO turns VALUES ('t1', 's1', '', 'done', 1, '', 'm1', 'm2', '', 'hi', '[]', '', 1000, 1001); +` + +// install writes the three legacy files the way an existing install holds +// them and returns their paths. mutate runs extra statements against a file +// before the import sees it. +func install(t *testing.T, mutate map[string]string) Sources { + t.Helper() + dir := t.TempDir() + src := Sources{ + StateStore: filepath.Join(dir, "archie.db-tasks.sqlite"), + EDA: filepath.Join(dir, "archie.db-eda.sqlite"), + Gateway: filepath.Join(dir, "archie.db-conversations.sqlite"), + } + for path, ddl := range map[string]string{src.StateStore: stateStoreDDL, src.EDA: edaDDL, src.Gateway: gatewayDDL} { + sqliteExec(t, path, ddl) + } + for path, stmt := range mutate { + switch path { + case "state": + sqliteExec(t, src.StateStore, stmt) + case "eda": + sqliteExec(t, src.EDA, stmt) + case "gateway": + sqliteExec(t, src.Gateway, stmt) + } + } + return src +} + +func sqliteExec(t *testing.T, path, stmt string) { + t.Helper() + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer func() { _ = db.Close() }() + if _, err := db.ExecContext(t.Context(), stmt); err != nil { + t.Fatalf("fixture %s: %v", filepath.Base(path), err) + } +} + +func hashFiles(t *testing.T, dir string) map[string][32]byte { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read dir: %v", err) + } + out := map[string][32]byte{} + for _, e := range entries { + b, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + t.Fatalf("read %s: %v", e.Name(), err) + } + out[e.Name()] = sha256.Sum256(b) + } + return out +} + +func target(t *testing.T) *pgxpool.Pool { + t.Helper() + pool, err := postgres.Open(t.Context(), pgtest.URL(t)) + if err != nil { + t.Fatalf("postgres.Open: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func migrated(t *testing.T) *pgxpool.Pool { + t.Helper() + pool := target(t) + if err := postgres.Migrate(t.Context(), pool, postgres.Migrations()); err != nil { + t.Fatalf("Migrate: %v", err) + } + return pool +} + +func count(t *testing.T, pool *pgxpool.Pool, table string) int { + t.Helper() + var n int + if err := pool.QueryRow(t.Context(), "SELECT count(*) FROM "+table).Scan(&n); err != nil { + t.Fatalf("count %s: %v", table, err) + } + return n +} + +func TestImportCopiesEverySourceAndLeavesThemUntouched(t *testing.T) { + src := install(t, nil) + before := hashFiles(t, filepath.Dir(src.StateStore)) + pool := target(t) + + report, err := Import(t.Context(), pool, src) + if err != nil { + t.Fatalf("Import: %v", err) + } + + if after := hashFiles(t, filepath.Dir(src.StateStore)); len(after) != len(before) { + t.Fatalf("source directory changed: %d files before, %d after", len(before), len(after)) + } else { + for name, sum := range before { + if after[name] != sum { + t.Errorf("source %s changed during import", name) + } + } + } + + done, err := postgres.ImportComplete(t.Context(), pool) + if err != nil || !done { + t.Fatalf("ImportComplete = %v, %v; want true", done, err) + } + + counts := map[string]int{ + "tasks": 2, "transitions": 1, "events": 3, "resources": 1, "resource_history": 1, + "channel_status": 1, "apply_status": 1, "config_snapshot": 1, "identities": 1, + "identity_aliases": 1, "identity_events": 1, "identity_subjects": 1, + "captures": 2, "mappings": 1, "bindings": 1, "binding_dispatches": 1, "playbook_dispatches": 1, + "tool_calls": 1, "sessions": 1, "messages": 2, "turns": 1, + } + for table, want := range counts { + if got := count(t, pool, table); got != want { + t.Errorf("%s rows = %d, want %d", table, got, want) + } + if got := report.Rows[table]; got != want { + t.Errorf("report rows[%s] = %d, want %d", table, got, want) + } + } + + ctx := t.Context() + var ( + reviewCursor int64 + value []byte + gotSecret string + headers string + result string + configured bool + ) + checks := []struct { + query string + dest any + }{ + {"SELECT review_cursor FROM tasks WHERE id = 1", &reviewCursor}, + {"SELECT value FROM resource_history WHERE id = 1", &value}, + {"SELECT secret FROM bindings WHERE id = 'rbind1'", &gotSecret}, + {"SELECT headers FROM captures WHERE id = 'rcap2'", &headers}, + {"SELECT result FROM tool_calls WHERE id = 'rtool1'", &result}, + {"SELECT configured FROM channel_status WHERE id = 'tg'", &configured}, + } + for _, c := range checks { + if err := pool.QueryRow(ctx, c.query).Scan(c.dest); err != nil { + t.Fatalf("%s: %v", c.query, err) + } + } + if reviewCursor != 3 { + t.Errorf("review_cursor = %d, want 3", reviewCursor) + } + if !bytes.Equal(value, []byte{0x00, 0xff, 0x10}) { + t.Errorf("resource_history.value = %x, want 00ff10", value) + } + if gotSecret != secret { + t.Errorf("binding secret = %q, want %q", gotSecret, secret) + } + if headers != "" || result != "exit 0" || !configured { + t.Errorf("headers %q, result %q, configured %v; want \"\", \"exit 0\", true", headers, result, configured) + } + + // Event order is (at, id) as the source had it, and the next event id + // follows AUTOINCREMENT's high-water mark rather than reusing one. + rows, err := pool.Query(ctx, "SELECT id FROM events ORDER BY at, id") + if err != nil { + t.Fatalf("events: %v", err) + } + var order []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + t.Fatalf("scan: %v", err) + } + order = append(order, id) + } + if want := []int64{1, 5, 2}; !equalInts(order, want) { + t.Errorf("event order = %v, want %v", order, want) + } + nextIDs := map[string]int64{"tasks": 10, "events": 6, "messages": 3} + for table, want := range nextIDs { + var got int64 + if err := pool.QueryRow(ctx, "SELECT nextval(pg_get_serial_sequence($1, 'id'))", table).Scan(&got); err != nil { + t.Fatalf("nextval %s: %v", table, err) + } + if got != want { + t.Errorf("next %s id = %d, want %d", table, got, want) + } + } + + var hits int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM messages WHERE search @@ to_tsquery('simple', 'hello')").Scan(&hits); err != nil { + t.Fatalf("search: %v", err) + } + if hits != 1 { + t.Errorf("search hits for an imported message = %d, want 1", hits) + } +} + +func equalInts(a, b []int64) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestImportRefuses(t *testing.T) { + tests := []struct { + name string + mutate map[string]string + seed func(t *testing.T, pool *pgxpool.Pool) + want []string + is error + }{ + { + name: "non-empty target", + seed: func(t *testing.T, pool *pgxpool.Pool) { + mustExec(t, pool, "INSERT INTO transitions (task_id, from_status, to_status) VALUES (1, 'a', 'b')") + }, + want: []string{"transitions", "not empty"}, + }, + { + name: "completed import", + seed: func(t *testing.T, pool *pgxpool.Pool) { + mustExec(t, pool, "INSERT INTO import_completion (id, sources, report) VALUES (1, 's', 'r')") + }, + want: []string{"already"}, + }, + { + name: "live state store", + seed: func(t *testing.T, pool *pgxpool.Pool) { holdOwnership(t, pool, postgres.OwnerStateStore) }, + want: []string{"state-store"}, + is: postgres.ErrOwned, + }, + { + name: "live gateway", + seed: func(t *testing.T, pool *pgxpool.Pool) { holdOwnership(t, pool, postgres.OwnerGateway) }, + want: []string{"gateway"}, + is: postgres.ErrOwned, + }, + { + name: "orphaned alias", + mutate: map[string]string{"state": "INSERT INTO identity_aliases VALUES ('ghost', 'id-404')"}, + want: []string{"identity_aliases", "ghost", "id-404"}, + }, + { + name: "duplicate bindings", + mutate: map[string]string{"eda": "INSERT INTO bindings VALUES ('rbind2', '2026-01-01 00:00:00.000Z', '', 'b2', 'o', 'r', '', 'gh', 'pending_approval', '2026-01-01 00:00:00.000Z', 1, 'implement')"}, + want: []string{"bindings", "gh", "rbind1", "rbind2"}, + }, + { + name: "microsecond collision", + mutate: map[string]string{"state": "INSERT INTO events (id, at, kind) VALUES (7, '2026-01-01T00:00:00.000000050Z', 'd')"}, + want: []string{"events", "7", "microsecond"}, + }, + { + name: "unparseable timestamp", + mutate: map[string]string{"eda": "UPDATE captures SET received_at = 'yesterday' WHERE id = 'rcap2'"}, + want: []string{"captures", "rcap2", "yesterday", "received_at"}, + }, + { + name: "null in a required column", + mutate: map[string]string{"state": "UPDATE tasks SET title = NULL WHERE id = 4"}, + want: []string{"tasks", "4", "title", "NULL"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := install(t, tt.mutate) + pool := migrated(t) + if tt.seed != nil { + tt.seed(t, pool) + } + _, err := Import(t.Context(), pool, src) + if err == nil { + t.Fatal("Import succeeded, want a refusal") + } + if tt.is != nil && !errors.Is(err, tt.is) { + t.Errorf("Import error %v, want %v", err, tt.is) + } + for _, w := range tt.want { + if !strings.Contains(err.Error(), w) { + t.Errorf("refusal %q does not name %q", err, w) + } + } + // Nothing of the import survives a refusal: the store stays + // unserveable rather than half-imported. + if got := count(t, pool, "tasks"); got != 0 { + t.Errorf("tasks rows after refusal = %d, want 0", got) + } + if got := count(t, pool, "import_completion"); tt.name != "completed import" && got != 0 { + t.Errorf("completion records after refusal = %d, want 0", got) + } + }) + } +} + +func mustExec(t *testing.T, pool *pgxpool.Pool, stmt string) { + t.Helper() + if _, err := pool.Exec(t.Context(), stmt); err != nil { + t.Fatalf("%s: %v", stmt, err) + } +} + +// holdOwnership claims name from a separate pool, as a live service would. +func holdOwnership(t *testing.T, pool *pgxpool.Pool, name string) { + t.Helper() + other, err := pgxpool.New(t.Context(), pool.Config().ConnString()) + if err != nil { + t.Fatalf("second pool: %v", err) + } + t.Cleanup(other.Close) + claim, err := postgres.AcquireOwnership(t.Context(), other, name) + if err != nil { + t.Fatalf("claim %s: %v", name, err) + } + t.Cleanup(func() { _ = claim.Release(t.Context()) }) +} + +// TestSiblingDomainDoesNotBlock pins that "non-empty" is per domain: a +// domain with no legacy source is not imported, so rows in its tables do not +// refuse the others. +func TestSiblingDomainDoesNotBlock(t *testing.T) { + src := install(t, nil) + src.Gateway = filepath.Join(t.TempDir(), "absent.sqlite") + pool := migrated(t) + mustExec(t, pool, "INSERT INTO sessions (session_id) VALUES ('live')") + if _, err := Import(t.Context(), pool, src); err != nil { + t.Fatalf("Import: %v", err) + } + if got := count(t, pool, "tasks"); got != 2 { + t.Fatalf("tasks rows = %d, want 2", got) + } +} + +func TestImportRefusesWithNoSources(t *testing.T) { + dir := t.TempDir() + src := Sources{StateStore: filepath.Join(dir, "a"), EDA: filepath.Join(dir, "b"), Gateway: filepath.Join(dir, "c")} + if _, err := Import(t.Context(), migrated(t), src); err == nil || !strings.Contains(err.Error(), "no legacy") { + t.Fatalf("Import = %v, want a refusal naming no legacy sources", err) + } +} + +func TestHasLegacyData(t *testing.T) { + dir := t.TempDir() + absent := Sources{StateStore: filepath.Join(dir, "a"), EDA: filepath.Join(dir, "b"), Gateway: filepath.Join(dir, "c")} + onlyGateway := absent + onlyGateway.Gateway = install(t, nil).Gateway + tests := []struct { + name string + src Sources + want bool + }{ + {"fresh install", absent, false}, + {"one source with rows", onlyGateway, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := HasLegacyData(t.Context(), tt.src) + if err != nil { + t.Fatalf("HasLegacyData: %v", err) + } + if got != tt.want { + t.Fatalf("HasLegacyData = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/infrastructure/legacyimport/load.go b/internal/infrastructure/legacyimport/load.go new file mode 100644 index 000000000..4d5096601 --- /dev/null +++ b/internal/infrastructure/legacyimport/load.go @@ -0,0 +1,288 @@ +package legacyimport + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "slices" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/samcharles93/archie-core/internal/infrastructure/legacyread" +) + +// droppedColumns are legacy columns with no target column by design: +// PocketBase gave every ledger row a random record id, and the Postgres +// ledgers are keyed by their natural unique key instead. +var droppedColumns = map[string]bool{ + "binding_dispatches.id": true, + "playbook_dispatches.id": true, +} + +// targetKeys override a table's legacy key where the target has no column +// for it. +var targetKeys = map[string][]string{ + "binding_dispatches": {"binding", "capture"}, + "playbook_dispatches": {"playbook_id", "playbook_version", "event_id", "action_id"}, +} + +// jsonColumns are PocketBase JSON fields, which may be NULL in the source and +// map to a NOT NULL text column defaulting to the empty string. unwrap marks the fields the legacy reader +// decoded from a JSON string (jsonTextToString) rather than returning raw. +var jsonColumns = map[string]struct{ unwrap bool }{ + "captures.headers": {}, + "mappings.fields": {}, + "tool_calls.args": {unwrap: true}, + "tool_calls.result": {unwrap: true}, +} + +// timeLayouts are every layout a legacy timestamp was written in: the +// fixed-width cursor layout, Go's RFC 3339 forms, PocketBase's autodate +// layout and SQLite's datetime('now'). +var timeLayouts = []string{ + "2006-01-02T15:04:05.000000000Z", + time.RFC3339Nano, + time.RFC3339, + "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02 15:04:05.999999999", +} + +type column struct { + name string + dataType string + identity bool + source int // index into the legacy row +} + +// loadedTable is what load wrote, for verify to check. +type loadedTable struct { + table legacyread.Table + rows legacyread.Rows + keys []string // target keys of the rows written, in source read order + keyCols []string + identity bool +} + +// load reads t from src, converts every row to the target's types and copies +// them in with their historical ids. A row it cannot convert is added to +// problems and nothing of the table is written. +func load(ctx context.Context, tx pgx.Tx, src *legacyread.Source, t legacyread.Table, problems *refusal) (loadedTable, error) { + cols, err := plan(ctx, tx, t, problems) + if err != nil || cols == nil { + return loadedTable{table: t}, err + } + rows, err := src.Read(ctx, t) + if err != nil { + return loadedTable{}, err + } + lt := loadedTable{table: t, rows: rows, keyCols: t.Key} + if k, ok := targetKeys[t.Name]; ok { + lt.keyCols = k + } + names := make([]string, len(cols)) + for i, c := range cols { + names[i] = c.name + lt.identity = lt.identity || c.identity + } + out, ok := convertRows(lt, cols, names, problems) + if !ok { + return lt, nil + } + for _, row := range out { + key := make([]string, len(lt.keyCols)) + for j, k := range lt.keyCols { + key[j] = fmt.Sprint(row[slices.Index(names, k)]) + } + lt.keys = append(lt.keys, strings.Join(key, "/")) + } + if len(out) > 0 { + if _, err := tx.CopyFrom(ctx, pgx.Identifier{t.Name}, names, pgx.CopyFromRows(out)); err != nil { + problems.add("%s: the target rejected the rows: %v", t.Name, err) + return lt, nil + } + } + if lt.identity { + return lt, advanceSequence(ctx, tx, src, t.Name) + } + return lt, nil +} + +// convertRows converts every row of lt, adding each value it cannot convert +// to problems. +func convertRows(lt loadedTable, cols []column, names []string, problems *refusal) ([][]any, bool) { + sourceKeys := lt.rows.Keys() + out := make([][]any, 0, len(lt.rows.Rows)) + ok := true + for i, row := range lt.rows.Rows { + converted := make([]any, len(cols)) + for j, c := range cols { + v, err := convert(lt.table.Name+"."+c.name, c.dataType, row[c.source]) + if err != nil { + problems.add("%s %s: column %s value %s: %v", lt.table.Name, sourceKeys[i], names[j], quote(row[c.source]), err) + ok = false + } + converted[j] = v + } + out = append(out, converted) + } + return out, ok +} + +// advanceSequence sets table's identity sequence to hand out the id the +// legacy AUTOINCREMENT would have, even for a table whose rows were all +// deleted. is_called = false makes that id the next one, which keeps +// AUTOINCREMENT's promise that a deleted top id is never reused. +func advanceSequence(ctx context.Context, tx pgx.Tx, src *legacyread.Source, table string) error { + seq, err := src.SequenceOf(ctx, table) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, "SELECT setval(pg_get_serial_sequence($1, 'id'), $2, false)", table, seq.Next); err != nil { + return fmt.Errorf("legacy import: advance %s sequence: %w", table, err) + } + return nil +} + +// plan matches the target table's columns to the legacy ones. A column on +// either side with no counterpart is a schema gap the import refuses to +// guess about; it returns nil columns then. +func plan(ctx context.Context, tx pgx.Tx, t legacyread.Table, problems *refusal) ([]column, error) { + rows, err := tx.Query(ctx, `SELECT column_name, data_type, is_identity = 'YES' + FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1 AND is_generated = 'NEVER' + ORDER BY ordinal_position`, t.Name) + if err != nil { + return nil, fmt.Errorf("legacy import: %s columns: %w", t.Name, err) + } + cols, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (column, error) { + var c column + err := row.Scan(&c.name, &c.dataType, &c.identity) + return c, err + }) + if err != nil { + return nil, fmt.Errorf("legacy import: %s columns: %w", t.Name, err) + } + gap := len(cols) == 0 + if gap { + problems.add("%s: no target table", t.Name) + } + targetNames := map[string]bool{} + for i := range cols { + targetNames[cols[i].name] = true + cols[i].source = slices.Index(t.Columns, cols[i].name) + if cols[i].source < 0 { + problems.add("%s.%s: the target column has no legacy column", t.Name, cols[i].name) + gap = true + } + } + for _, c := range t.Columns { + if !targetNames[c] && !droppedColumns[t.Name+"."+c] { + problems.add("%s.%s: the legacy column has no target column", t.Name, c) + gap = true + } + } + if gap { + return nil, nil + } + return cols, nil +} + +// convert maps one SQLite value to the Go value pgx writes to a column of +// dataType. It never coerces: a value the column cannot hold is an error +// naming the rule it breaks. +func convert(qualified, dataType string, v any) (any, error) { + if j, ok := jsonColumns[qualified]; ok { + return convertJSON(v, j.unwrap) + } + if v == nil { + return nil, fmt.Errorf("NULL in a NOT NULL %s column", dataType) + } + conv, ok := converters[dataType] + if !ok { + return nil, fmt.Errorf("unsupported target type %s", dataType) + } + return conv(v) +} + +func convertJSON(v any, unwrap bool) (any, error) { + switch s := v.(type) { + case nil: + return "", nil + case string: + if unwrap { + var decoded string + if json.Unmarshal([]byte(s), &decoded) == nil { + return decoded, nil + } + } + return s, nil + } + return nil, errors.New("text: not a string") +} + +// converters are keyed by information_schema data_type. +var converters = map[string]func(any) (any, error){ + "text": func(v any) (any, error) { + if s, ok := v.(string); ok { + return s, nil + } + return nil, errors.New("text: not a string") + }, + "bigint": func(v any) (any, error) { + switch n := v.(type) { + case int64: + return n, nil + case float64: + if n == math.Trunc(n) && math.Abs(n) < 1<<63 { + return int64(n), nil + } + } + return nil, errors.New("bigint: not an integer") + }, + "boolean": func(v any) (any, error) { + switch b := v.(type) { + case bool: + return b, nil + case int64: + if b == 0 || b == 1 { + return b == 1, nil + } + } + return nil, errors.New("boolean: not 0 or 1") + }, + "bytea": func(v any) (any, error) { + switch b := v.(type) { + case []byte: + return b, nil + case string: + return []byte(b), nil + } + return nil, errors.New("bytea: not a blob") + }, + "timestamp with time zone": func(v any) (any, error) { + if s, ok := v.(string); ok { + for _, layout := range timeLayouts { + if at, err := time.Parse(layout, s); err == nil { + return at.UTC(), nil + } + } + } + return nil, errors.New("timestamptz: not a timestamp in any legacy layout") + }, +} + +func quote(v any) string { + switch s := v.(type) { + case nil: + return "NULL" + case string: + return fmt.Sprintf("%q", s) + case []byte: + return fmt.Sprintf("x'%x'", s) + } + return fmt.Sprint(v) +} diff --git a/internal/infrastructure/legacyimport/verify.go b/internal/infrastructure/legacyimport/verify.go new file mode 100644 index 000000000..6fa4e9bde --- /dev/null +++ b/internal/infrastructure/legacyimport/verify.go @@ -0,0 +1,171 @@ +package legacyimport + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/jackc/pgx/v5" + + "github.com/samcharles93/archie-core/internal/infrastructure/legacyread" +) + +// blobColumns are the columns whose bytes must survive exactly: resource +// documents, and binding secrets, which stay decryptable by the configured +// keyring only if the envelope is untouched. +var blobColumns = map[string]string{ + "resources": "value", + "resource_history": "value", + "bindings": "secret", +} + +// verify reads back what load wrote for lt and compares it with the source: +// the key set (a count passes on a swallowed row when ids have gaps), the +// (at, id) event order, blob bytes and the identity sequence. +func verify(ctx context.Context, tx pgx.Tx, lt loadedTable, problems *refusal) error { + name := lt.table.Name + ident := pgx.Identifier{name}.Sanitize() + keyExprs := make([]string, len(lt.keyCols)) + for i, k := range lt.keyCols { + keyExprs[i] = pgx.Identifier{k}.Sanitize() + "::text" + } + keyExpr := "concat_ws('/', " + strings.Join(keyExprs, ", ") + ")" + + targetKeys, err := strs(ctx, tx, "SELECT "+keyExpr+" FROM "+ident) + if err != nil { + return fmt.Errorf("legacy import: verify %s: %w", name, err) + } + if len(targetKeys) != len(lt.keys) { + problems.add("%s: %d rows read, %d rows in the target", name, len(lt.keys), len(targetKeys)) + } + if d := legacyread.CompareKeys(lt.keys, targetKeys); len(d.Missing)+len(d.Extra) > 0 { + problems.add("%s: rows missing from the target %v, rows only in the target %v", name, d.Missing, d.Extra) + } + + if name == "events" { + if err := verifyEventOrder(ctx, tx, lt, problems); err != nil { + return err + } + } + if col, ok := blobColumns[name]; ok { + if err := verifyBlobs(ctx, tx, lt, col, keyExpr, problems); err != nil { + return err + } + } + if lt.identity { + return verifySequence(ctx, tx, name, problems) + } + return nil +} + +// verifySequence asserts the next id the sequence hands out is above every +// imported id, reading its state rather than calling nextval. +func verifySequence(ctx context.Context, tx pgx.Tx, name string, problems *refusal) error { + var seq string + var maxID int64 + if err := tx.QueryRow(ctx, "SELECT pg_get_serial_sequence($1, 'id'), (SELECT COALESCE(max(id), 0) FROM "+ + pgx.Identifier{name}.Sanitize()+")", name).Scan(&seq, &maxID); err != nil { + return fmt.Errorf("legacy import: verify %s sequence: %w", name, err) + } + var ( + last int64 + called bool + ) + // seq comes from pg_get_serial_sequence, which returns it quoted. + if err := tx.QueryRow(ctx, "SELECT last_value, is_called FROM "+seq).Scan(&last, &called); err != nil { + return fmt.Errorf("legacy import: verify %s sequence: %w", name, err) + } + next := last + if called { + next++ + } + if next <= maxID { + problems.add("%s: next id %d would reuse an imported id (max %d)", name, next, maxID) + } + return nil +} + +// verifyEventOrder compares the ids in (at, id) order on both sides. The +// source was read in that order; a difference means microsecond storage +// reordered history, and a cursor reader would skip a row. +func verifyEventOrder(ctx context.Context, tx pgx.Tx, lt loadedTable, problems *refusal) error { + source := make([]int64, len(lt.keys)) + for i, k := range lt.keys { + id, err := strconv.ParseInt(k, 10, 64) + if err != nil { + return fmt.Errorf("legacy import: event key %q: %w", k, err) + } + source[i] = id + } + rows, err := tx.Query(ctx, "SELECT id FROM events ORDER BY at, id") + if err != nil { + return fmt.Errorf("legacy import: verify events order: %w", err) + } + target, err := pgx.CollectRows(rows, pgx.RowTo[int64]) + if err != nil { + return fmt.Errorf("legacy import: verify events order: %w", err) + } + if d := legacyread.CompareOrder(source, target); d != nil { + problems.add("events: (at, id) order differs at position %d: source id %d, target id %d", d.Position, d.Source, d.Target) + } + return nil +} + +func verifyBlobs(ctx context.Context, tx pgx.Tx, lt loadedTable, col, keyExpr string, problems *refusal) error { + source := map[string][]byte{} + for i, k := range lt.keys { + switch v := lt.rows.Value(i, col).(type) { + case []byte: + source[k] = v + case string: + source[k] = []byte(v) + } + } + rows, err := tx.Query(ctx, fmt.Sprintf("SELECT %s, %s FROM %s", keyExpr, + pgx.Identifier{col}.Sanitize(), pgx.Identifier{lt.table.Name}.Sanitize())) + if err != nil { + return fmt.Errorf("legacy import: verify %s.%s: %w", lt.table.Name, col, err) + } + target := map[string][]byte{} + for rows.Next() { + var ( + k string + v []byte + ) + if err := rows.Scan(&k, &v); err != nil { + rows.Close() + return fmt.Errorf("legacy import: verify %s.%s: %w", lt.table.Name, col, err) + } + target[k] = v + } + if err := rows.Err(); err != nil { + return fmt.Errorf("legacy import: verify %s.%s: %w", lt.table.Name, col, err) + } + if diff := legacyread.CompareBlobs(source, target); len(diff) > 0 { + problems.add("%s.%s: bytes differ from the source for %v", lt.table.Name, col, diff) + } + return nil +} + +// unindexedMessages returns the source message ids the FTS5 index missed. +func unindexedMessages(ctx context.Context, gw *legacyread.Source, loaded []loadedTable) ([]string, error) { + indexed, err := gw.IndexedMessageIDs(ctx) + if err != nil { + return nil, err + } + for _, lt := range loaded { + if lt.table.Name == "messages" { + return legacyread.CompareKeys(indexed, lt.keys).Extra, nil + } + } + return nil, nil +} + +func strs(ctx context.Context, tx pgx.Tx, query string) ([]string, error) { + rows, err := tx.Query(ctx, query) + if err != nil { + return nil, err + } + return pgx.CollectRows(rows, pgx.RowTo[string]) +} diff --git a/internal/infrastructure/legacyread/presence.go b/internal/infrastructure/legacyread/presence.go new file mode 100644 index 000000000..5697c8f5f --- /dev/null +++ b/internal/infrastructure/legacyread/presence.go @@ -0,0 +1,55 @@ +package legacyread + +import ( + "context" + "errors" + "fmt" + "os" +) + +// HasData reports whether the legacy file at path holds any row in tables. A +// missing file, and a table the file never created, hold nothing. It is the +// legacy half of the boot gate: a Postgres database without a completed import +// must not serve while this is true for any source. +func HasData(ctx context.Context, path string, tables []Table) (bool, error) { + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return false, nil + } + src, err := Open(ctx, path) + if err != nil { + return false, err + } + defer func() { _ = src.Close() }() + for _, t := range tables { + var present bool + if err := src.db.QueryRowContext(ctx, + "SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)", t.Name).Scan(&present); err != nil { + return false, fmt.Errorf("legacyread: %s: %w", path, err) + } + if !present { + continue + } + var rows bool + if err := src.db.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM "+t.Name+")").Scan(&rows); err != nil { + return false, fmt.Errorf("legacyread: %s: %s: %w", path, t.Name, err) + } + if rows { + return true, nil + } + } + return false, nil +} + +// SequenceOf returns the state of table's AUTOINCREMENT id column. +func (s *Source) SequenceOf(ctx context.Context, table string) (Sequence, error) { + seq := Sequence{Table: table} + var high int64 + err := s.db.QueryRowContext(ctx, fmt.Sprintf( + "SELECT COALESCE((SELECT max(id) FROM %[1]s), 0), COALESCE((SELECT seq FROM sqlite_sequence WHERE name = '%[1]s'), 0)", + table)).Scan(&seq.MaxID, &high) + if err != nil { + return Sequence{}, fmt.Errorf("legacyread: sequence %s: %w", table, err) + } + seq.Next = max(seq.MaxID, high) + 1 + return seq, nil +} diff --git a/internal/infrastructure/legacyread/presence_test.go b/internal/infrastructure/legacyread/presence_test.go new file mode 100644 index 000000000..ccc2018c1 --- /dev/null +++ b/internal/infrastructure/legacyread/presence_test.go @@ -0,0 +1,46 @@ +package legacyread + +import ( + "path/filepath" + "testing" +) + +func TestHasData(t *testing.T) { + emptyGateway := `CREATE TABLE sessions (session_id TEXT PRIMARY KEY); +CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT);` + tests := []struct { + name string + path func(t *testing.T) string + tables []Table + want bool + }{ + {"missing file", func(t *testing.T) string { return filepath.Join(t.TempDir(), "absent.sqlite") }, GatewayTables, false}, + {"tables without rows", func(t *testing.T) string { return fixture(t, "gw.sqlite", emptyGateway) }, GatewayTables, false}, + {"rows present", func(t *testing.T) string { return fixture(t, "gw.sqlite", gatewayDDL) }, GatewayTables, true}, + {"rows in the state store", func(t *testing.T) string { return fixture(t, "archie.db", stateStoreDDL) }, StateStoreTables, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := HasData(t.Context(), tt.path(t), tt.tables) + if err != nil { + t.Fatalf("HasData: %v", err) + } + if got != tt.want { + t.Fatalf("HasData = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSequenceOfMessages(t *testing.T) { + path := fixture(t, "gw.sqlite", gatewayDDL) + exec(t, path, `DELETE FROM messages WHERE message_id = 'm2'`) + src := openSource(t, path) + got, err := src.SequenceOf(t.Context(), "messages") + if err != nil { + t.Fatalf("SequenceOf: %v", err) + } + if want := (Sequence{Table: "messages", MaxID: 1, Next: 3}); got != want { + t.Fatalf("SequenceOf(messages) = %+v, want %+v", got, want) + } +} diff --git a/internal/infrastructure/legacyread/verify.go b/internal/infrastructure/legacyread/verify.go index d3ecc1435..6fb2f4d68 100644 --- a/internal/infrastructure/legacyread/verify.go +++ b/internal/infrastructure/legacyread/verify.go @@ -242,15 +242,10 @@ var sequenceTables = []string{"tasks", "transitions", "events", "resource_histor func (s *Source) Sequences(ctx context.Context) ([]Sequence, error) { out := make([]Sequence, 0, len(sequenceTables)) for _, table := range sequenceTables { - seq := Sequence{Table: table} - var high int64 - err := s.db.QueryRowContext(ctx, fmt.Sprintf( - "SELECT COALESCE((SELECT max(id) FROM %[1]s), 0), COALESCE((SELECT seq FROM sqlite_sequence WHERE name = '%[1]s'), 0)", - table)).Scan(&seq.MaxID, &high) + seq, err := s.SequenceOf(ctx, table) if err != nil { - return nil, fmt.Errorf("legacyread: sequence %s: %w", table, err) + return nil, err } - seq.Next = max(seq.MaxID, high) + 1 out = append(out, seq) } return out, nil diff --git a/internal/infrastructure/postgres/import_completion.go b/internal/infrastructure/postgres/import_completion.go new file mode 100644 index 000000000..32071fd71 --- /dev/null +++ b/internal/infrastructure/postgres/import_completion.go @@ -0,0 +1,29 @@ +package postgres + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb" +) + +// Service owner names: every Archie process that serves this database holds +// one of these claims for its whole life, so an offline rewrite checks them +// all. +const ( + OwnerStateStore = "state-store" + OwnerGateway = "gateway" +) + +// ImportComplete reports whether the one-time legacy import committed and +// verified against this migrated database. Serving over non-empty legacy +// files is refused without it. +func ImportComplete(ctx context.Context, pool *pgxpool.Pool) (bool, error) { + done, err := postgresdb.New(pool).ImportCompleted(ctx) + if err != nil { + return false, fmt.Errorf("postgres: import completion: %w", err) + } + return done, nil +} diff --git a/internal/infrastructure/postgres/import_completion_test.go b/internal/infrastructure/postgres/import_completion_test.go new file mode 100644 index 000000000..cd6e459df --- /dev/null +++ b/internal/infrastructure/postgres/import_completion_test.go @@ -0,0 +1,39 @@ +package postgres + +import ( + "testing" + + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" +) + +func TestImportComplete(t *testing.T) { + tests := []struct { + name string + record bool + want bool + }{ + {name: "migrated database without a record", want: false}, + {name: "committed record", record: true, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pool := openPool(t, pgtest.URL(t)) + if err := Migrate(t.Context(), pool, Migrations()); err != nil { + t.Fatalf("Migrate: %v", err) + } + if tt.record { + if _, err := pool.Exec(t.Context(), + "INSERT INTO import_completion (id, sources, report) VALUES (1, 's', 'r')"); err != nil { + t.Fatalf("seed record: %v", err) + } + } + got, err := ImportComplete(t.Context(), pool) + if err != nil { + t.Fatalf("ImportComplete: %v", err) + } + if got != tt.want { + t.Fatalf("ImportComplete = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/infrastructure/postgres/migrations/0005_import_completion.sql b/internal/infrastructure/postgres/migrations/0005_import_completion.sql new file mode 100644 index 000000000..f44a91023 --- /dev/null +++ b/internal/infrastructure/postgres/migrations/0005_import_completion.sql @@ -0,0 +1,19 @@ +-- import_completion records that the one-time import of an existing install's +-- SQLite stores committed and passed verification. It is written in the same +-- transaction as the imported rows, so a record exists exactly when the data +-- does. An empty database is not evidence that an import succeeded, only that +-- nothing has written to it yet, so serving over non-empty legacy files +-- requires this row. + +-- +goose Up + +CREATE TABLE import_completion ( + id bigint PRIMARY KEY CHECK (id = 1), + completed_at timestamptz NOT NULL DEFAULT now(), + sources text NOT NULL, + report text NOT NULL +); + +-- +goose Down + +DROP TABLE import_completion; diff --git a/internal/infrastructure/postgres/postgresdb/import.sql.go b/internal/infrastructure/postgres/postgresdb/import.sql.go new file mode 100644 index 000000000..c36c97386 --- /dev/null +++ b/internal/infrastructure/postgres/postgresdb/import.sql.go @@ -0,0 +1,37 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: import.sql + +package postgresdb + +import ( + "context" +) + +const importCompleted = `-- name: ImportCompleted :one +SELECT EXISTS (SELECT 1 FROM import_completion) +` + +func (q *Queries) ImportCompleted(ctx context.Context) (bool, error) { + row := q.db.QueryRow(ctx, importCompleted) + var exists bool + err := row.Scan(&exists) + return exists, err +} + +const insertImportCompletion = `-- name: InsertImportCompletion :exec + +INSERT INTO import_completion (id, sources, report) VALUES (1, $1, $2) +` + +type InsertImportCompletionParams struct { + Sources string + Report string +} + +// The one-time legacy import's completion record. +func (q *Queries) InsertImportCompletion(ctx context.Context, arg InsertImportCompletionParams) error { + _, err := q.db.Exec(ctx, insertImportCompletion, arg.Sources, arg.Report) + return err +} diff --git a/internal/infrastructure/postgres/postgresdb/models.go b/internal/infrastructure/postgres/postgresdb/models.go index 021a99135..abb6972af 100644 --- a/internal/infrastructure/postgres/postgresdb/models.go +++ b/internal/infrastructure/postgres/postgresdb/models.go @@ -119,6 +119,13 @@ type IdentitySubject struct { BoundAt time.Time } +type ImportCompletion struct { + ID int64 + CompletedAt time.Time + Sources string + Report string +} + type Mapping struct { ID string Name string diff --git a/internal/infrastructure/postgres/queries/import.sql b/internal/infrastructure/postgres/queries/import.sql new file mode 100644 index 000000000..0a741eaf2 --- /dev/null +++ b/internal/infrastructure/postgres/queries/import.sql @@ -0,0 +1,7 @@ +-- The one-time legacy import's completion record. + +-- name: InsertImportCompletion :exec +INSERT INTO import_completion (id, sources, report) VALUES (1, $1, $2); + +-- name: ImportCompleted :one +SELECT EXISTS (SELECT 1 FROM import_completion);