diff --git a/cmd/archie-state-store/import_test.go b/cmd/archie-state-store/import_test.go new file mode 100644 index 000000000..f0a61458c --- /dev/null +++ b/cmd/archie-state-store/import_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/infrastructure/postgres" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" +) + +func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) } + +// TestRecoveryImportMovesTheInstallOnce drives the import the way an operator +// does: the configured db_path locates every legacy file, and a second run +// against the same database is refused because the first one completed. +func TestRecoveryImportMovesTheInstallOnce(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "archie.db") + st := openTaskStore(t, dbPath+"-tasks.sqlite") + seedTask(t, st, "carried over") + if err := st.Close(); err != nil { + t.Fatalf("close task store: %v", err) + } + url := pgtest.URL(t) + + code, stdout, stderr := runRecoveryCmd(t, "import", "-db-path", dbPath, "-database-url", url) + if code != 0 { + t.Fatalf("import exited %d; stderr = %q", code, stderr) + } + if !strings.Contains(stdout, "tasks=1") { + t.Errorf("import summary %q does not report the imported task", stdout) + } + + pool, err := postgres.Open(t.Context(), url) + if err != nil { + t.Fatalf("postgres.Open: %v", err) + } + defer pool.Close() + var title string + if err := pool.QueryRow(t.Context(), "SELECT title FROM tasks").Scan(&title); err != nil || title != "carried over" { + t.Fatalf("imported task title = %q, %v; want %q", title, err, "carried over") + } + + code, _, stderr = runRecoveryCmd(t, "import", "-db-path", dbPath, "-database-url", url) + if code != 1 || !strings.Contains(stderr, "already holds a completed import") { + t.Fatalf("second import exited %d, stderr %q; want 1 naming the completed import", code, stderr) + } +} diff --git a/cmd/archie-state-store/main.go b/cmd/archie-state-store/main.go index c664fa746..c95b8f2c2 100644 --- a/cmd/archie-state-store/main.go +++ b/cmd/archie-state-store/main.go @@ -40,7 +40,7 @@ func run() int { flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)") flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)") flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store -h\n\n") + fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store -h\n\n") flag.PrintDefaults() } flag.Parse() diff --git a/cmd/archie-state-store/recovery.go b/cmd/archie-state-store/recovery.go index 4d115a338..d96cfe466 100644 --- a/cmd/archie-state-store/recovery.go +++ b/cmd/archie-state-store/recovery.go @@ -31,6 +31,9 @@ owns it, while backup takes a consistent snapshot of a serving store. configuration the daemon would boot with rollback -db FILE -kind KIND replay an earlier revision of a stored [-revision N] resource through the ordinary replace + import [-db-path PATH] copy the legacy SQLite stores named from + [-database-url URL] db_path into an empty Postgres database, + [-config FILE] once; refuses while any Archie service runs ` // runRecovery parses and performs one offline recovery command, reporting the @@ -44,7 +47,8 @@ func runRecovery(args []string, stdout, stderr io.Writer) int { command := args[0] options := archied.StateStoreRecoveryOptions{Operation: command} switch command { - case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback: + case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback, + archied.RecoveryImport: default: fmt.Fprintf(stderr, "archie-state-store: unknown command %q\n\n", command) fmt.Fprint(stderr, recoveryUsage) @@ -59,9 +63,14 @@ func runRecovery(args []string, stdout, stderr io.Writer) int { flags.StringVar(&options.From, "from", "", "snapshot file restore reads") flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays") flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)") - // Only validate asks a question about the process rather than the file, so - // only validate takes the configuration the process would boot with. - if command == archied.RecoveryValidate { + // validate asks a question about the process rather than the file, and + // import reads db_path and database_url from it, so only those two take + // the configuration the process would boot with. + if command == archied.RecoveryImport { + flags.StringVar(&options.DBPath, "db-path", "", "configured db_path the legacy files are named from (default: the configuration's)") + flags.StringVar(&options.DatabaseURL, "database-url", "", "Postgres URL to import into (default: the configuration's database_url)") + } + if command == archied.RecoveryValidate || command == archied.RecoveryImport { flags.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory the daemon boots with") flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") } diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go index 0ab53025c..436e18306 100644 --- a/cmd/archie-state-store/recovery_test.go +++ b/cmd/archie-state-store/recovery_test.go @@ -356,7 +356,7 @@ func TestRecoveryValidateRunsTheGateBootRuns(t *testing.T) { // path in the same binary exits 0 for the same request, so a script probing the // recovery surface must not read a requested help as a failure. func TestRecoveryHelpExitsZero(t *testing.T) { - commands := []string{archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback} + commands := []string{archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback, archied.RecoveryImport} for _, command := range commands { code, _, stderr := runRecoveryCmd(t, command, "-h") if code != 0 { diff --git a/internal/app/archied/state_store_import.go b/internal/app/archied/state_store_import.go new file mode 100644 index 000000000..dc7f165be --- /dev/null +++ b/internal/app/archied/state_store_import.go @@ -0,0 +1,56 @@ +package archied + +import ( + "context" + "errors" + "fmt" + + "github.com/samcharles93/archie-core/internal/infrastructure/legacyimport" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres" +) + +// legacySources names an install's three legacy SQLite files from its +// configured db_path, the same derivation the legacy openers use, so an +// operator cannot import one store and forget another. +func legacySources(dbPath string) legacyimport.Sources { + return legacyimport.Sources{ + StateStore: taskDBPath(dbPath), + EDA: edaDBPath(dbPath), + Gateway: conversationDBPath(dbPath), + } +} + +// importLegacy runs the one-time import of the legacy files into Postgres. +// It is an offline command of the State Store binary, not of archied: offline +// maintenance of a store belongs to the process that owns it. +func importLegacy(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + dbPath, url := options.DBPath, options.DatabaseURL + if dbPath == "" || url == "" { + cfg, err := bootConfig(ctx, options) + if err != nil { + return "", err + } + if dbPath == "" { + dbPath = cfg.DBPath + } + if url == "" { + url = cfg.DatabaseURL + } + } + if dbPath == "" { + return "", errors.New("import requires db_path (-db-path or the configuration)") + } + if url == "" { + return "", errors.New("import requires database_url (-database-url or the configuration)") + } + pool, err := postgres.Open(ctx, url) + if err != nil { + return "", err + } + defer pool.Close() + report, err := legacyimport.Import(ctx, pool, legacySources(dbPath)) + if err != nil { + return "", err + } + return fmt.Sprintf("imported %s into Postgres and recorded the completion: %s", dbPath, report), nil +} diff --git a/internal/app/archied/state_store_recovery.go b/internal/app/archied/state_store_recovery.go index fe4636726..04ee111c5 100644 --- a/internal/app/archied/state_store_recovery.go +++ b/internal/app/archied/state_store_recovery.go @@ -46,6 +46,7 @@ const ( RecoveryRestore = "restore" RecoveryValidate = "validate" RecoveryRollback = "rollback" + RecoveryImport = "import" ) // StateStoreRecoveryOptions are the process inputs for one offline recovery @@ -71,6 +72,11 @@ type StateStoreRecoveryOptions struct { // Revision is the revision rollback replays. Zero means the newest // revision older than the one the resource carries now. Revision int64 + // DBPath and DatabaseURL are import's source and target: the configured + // db_path every legacy file is named from, and the Postgres URL. Either + // left empty is read from the configuration. + DBPath string + DatabaseURL string } // RunStateStoreRecovery performs one offline operation on the task database and @@ -81,18 +87,13 @@ func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOption if options.DB == "" || options.Out == "" { return "", errors.New("backup requires -db and -out") } - if err := store.Backup(ctx, options.DB, options.Out); err != nil { - return "", err - } - return fmt.Sprintf("backed up %s to %s", options.DB, options.Out), nil + return summarise(store.Backup(ctx, options.DB, options.Out), "backed up %s to %s", options.DB, options.Out) case RecoveryRestore: if options.DB == "" || options.From == "" { return "", errors.New("restore requires -db and -from") } - if err := store.Restore(ctx, options.DB, options.From); err != nil { - return "", err - } - return fmt.Sprintf("restored %s from %s; start the State Store again", options.DB, options.From), nil + return summarise(store.Restore(ctx, options.DB, options.From), + "restored %s from %s; start the State Store again", options.DB, options.From) case RecoveryValidate: if options.DB == "" { return "", errors.New("validate requires -db") @@ -103,11 +104,21 @@ func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOption return "", errors.New("rollback requires -db and -kind") } return rollbackResource(ctx, options) + case RecoveryImport: + return importLegacy(ctx, options) default: return "", fmt.Errorf("unknown recovery command %q", options.Operation) } } +// summarise is an operation's stdout line, or its error. +func summarise(err error, format string, args ...any) (string, error) { + if err != nil { + return "", err + } + return fmt.Sprintf(format, args...), nil +} + // validateStore answers the question an operator actually has: would archied // start against this file. It runs the checks that stop it, in the order boot // meets them. diff --git a/internal/infrastructure/legacyimport/legacyimport.go b/internal/infrastructure/legacyimport/legacyimport.go new file mode 100644 index 000000000..49e5c6e7f --- /dev/null +++ b/internal/infrastructure/legacyimport/legacyimport.go @@ -0,0 +1,387 @@ +// Package legacyimport moves an existing install's SQLite stores (the State +// Store's task database, the PocketBase event store and the Gateway +// conversation store) into Postgres, once. +// +// The import is one transaction: it refuses a target that already holds data +// in a domain it imports, refuses rather than skips a row the target cannot +// hold, verifies what it wrote against what it read, and only then records a +// completion row and commits. A refusal at any point leaves the target as it +// was, migrated and empty, so a failed import is never serveable. +// +// It reads through legacyread, which never writes to the source files, and is +// deleted together with it. +package legacyimport + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "maps" + "os" + "slices" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/samcharles93/archie-core/internal/infrastructure/legacyread" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb" +) + +// ErrRefused wraps every refusal. The message names each offending table, +// row, value and rule. +var ErrRefused = errors.New("legacy import refused") + +// Sources are the legacy files. A path whose file does not exist means that +// domain has nothing to import. +type Sources struct { + StateStore string + EDA string + Gateway string +} + +type domain struct { + name string + path string + tables []legacyread.Table +} + +func (s Sources) domains() []domain { + return []domain{ + {"state-store", s.StateStore, legacyread.StateStoreTables}, + {"eda", s.EDA, legacyread.EDATables}, + {"gateway", s.Gateway, legacyread.GatewayTables}, + } +} + +// present returns the domains whose source file exists. +func (s Sources) present() ([]domain, error) { + var out []domain + for _, d := range s.domains() { + if d.path == "" { + continue + } + _, err := os.Stat(d.path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return nil, fmt.Errorf("legacy import: %w", err) + } + out = append(out, d) + } + return out, nil +} + +// HasLegacyData reports whether any legacy source holds a row. With +// postgres.ImportComplete it is what decides whether a Postgres database may +// serve: legacy data without a completed import must be refused. +func HasLegacyData(ctx context.Context, src Sources) (bool, error) { + for _, d := range src.domains() { + if d.path == "" { + continue + } + has, err := legacyread.HasData(ctx, d.path, d.tables) + if err != nil || has { + return has, err + } + } + return false, nil +} + +// Report is what an import wrote. +type Report struct { + // Rows is each imported table's row count. + Rows map[string]int + // UnindexedMessages are Gateway message ids the source's FTS5 index did + // not cover. The target derives its search column from every row, so + // these become searchable; the count says the source index was stale. + UnindexedMessages []string +} + +func (r Report) String() string { + var b strings.Builder + for _, table := range slices.Sorted(maps.Keys(r.Rows)) { + fmt.Fprintf(&b, "%s=%d ", table, r.Rows[table]) + } + fmt.Fprintf(&b, "unindexed_messages=%d", len(r.UnindexedMessages)) + return b.String() +} + +// refusal collects every problem found before giving up, so an operator fixes +// a source in one pass rather than one row per attempt. +type refusal []string + +func (r *refusal) add(format string, args ...any) { *r = append(*r, fmt.Sprintf(format, args...)) } + +func (r *refusal) err() error { + if len(*r) == 0 { + return nil + } + return fmt.Errorf("%w:\n %s", ErrRefused, strings.Join(*r, "\n ")) +} + +// Import runs the one-time import from src into the database behind pool. +// Every service that serves the database must be stopped: the import holds +// each service's ownership claim for its whole run and refuses if any is +// taken. +func Import(ctx context.Context, pool *pgxpool.Pool, src Sources) (Report, error) { + present, err := src.present() + if err != nil { + return Report{}, err + } + if len(present) == 0 { + return Report{}, fmt.Errorf("%w: no legacy source files found (%s, %s, %s)", ErrRefused, src.StateStore, src.EDA, src.Gateway) + } + before, err := hashSources(present) + if err != nil { + return Report{}, err + } + if err := postgres.Migrate(ctx, pool, postgres.Migrations()); err != nil { + return Report{}, err + } + release, err := claimServices(ctx, pool) + defer release() + if err != nil { + return Report{}, err + } + sources, closeSources, err := openSources(ctx, present) + defer closeSources() + if err != nil { + return Report{}, err + } + + var problems refusal + if err := checkSources(ctx, sources, &problems); err != nil { + return Report{}, err + } + if err := problems.err(); err != nil { + return Report{}, err + } + + tx, err := pool.Begin(ctx) + if err != nil { + return Report{}, fmt.Errorf("legacy import: begin: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + report, err := importInto(ctx, tx, present, sources, &problems) + if err != nil { + return Report{}, err + } + if err := checkUnchanged(before, present, &problems); err != nil { + return Report{}, err + } + if err := problems.err(); err != nil { + return Report{}, err + } + if err := complete(ctx, tx, present, report); err != nil { + return Report{}, err + } + return report, nil +} + +// checkUnchanged refuses when a source's bytes moved during the import: a +// writer that was not stopped would leave the target short of its last writes. +func checkUnchanged(before map[string][32]byte, present []domain, problems *refusal) error { + after, err := hashSources(present) + if err != nil { + return err + } + for path, sum := range before { + if after[path] != sum { + problems.add("%s changed while it was read: stop the process writing it and import again", path) + } + } + return nil +} + +// complete writes the completion record and commits, so the record exists +// exactly when the imported rows do. +func complete(ctx context.Context, tx pgx.Tx, present []domain, report Report) error { + paths := make([]string, len(present)) + for i, d := range present { + paths[i] = d.path + } + if err := postgresdb.New(tx).InsertImportCompletion(ctx, postgresdb.InsertImportCompletionParams{ + Sources: strings.Join(paths, "\n"), + Report: report.String(), + }); err != nil { + return fmt.Errorf("legacy import: completion record: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("legacy import: commit: %w", err) + } + return nil +} + +// claimServices takes every service's ownership claim without waiting. The +// returned release is safe to call whether or not the claims were taken. +func claimServices(ctx context.Context, pool *pgxpool.Pool) (func(), error) { + var claims []*postgres.Ownership + release := func() { + for _, c := range claims { + _ = c.Release(context.WithoutCancel(ctx)) + } + } + for _, name := range []string{postgres.OwnerStateStore, postgres.OwnerGateway} { + claim, err := postgres.AcquireOwnership(ctx, pool, name) + if err != nil { + return release, fmt.Errorf("legacy import: stop every Archie service first: %w", err) + } + claims = append(claims, claim) + } + return release, nil +} + +// openSources opens each present source read-only, keyed by domain name. +func openSources(ctx context.Context, present []domain) (map[string]*legacyread.Source, func(), error) { + sources := map[string]*legacyread.Source{} + closeAll := func() { + for _, s := range sources { + _ = s.Close() + } + } + for _, d := range present { + s, err := legacyread.Open(ctx, d.path) + if err != nil { + return nil, closeAll, err + } + sources[d.name] = s + } + return sources, closeAll, nil +} + +// importInto loads and verifies every present domain inside tx, adding to +// problems anything that must stop the commit. +func importInto(ctx context.Context, tx pgx.Tx, present []domain, sources map[string]*legacyread.Source, problems *refusal) (Report, error) { + if err := checkTarget(ctx, tx, present, problems); err != nil { + return Report{}, err + } + if err := problems.err(); err != nil { + return Report{}, err + } + report := Report{Rows: map[string]int{}} + var loaded []loadedTable + for _, d := range present { + for _, t := range d.tables { + lt, err := load(ctx, tx, sources[d.name], t, problems) + if err != nil { + return Report{}, err + } + loaded = append(loaded, lt) + report.Rows[t.Name] = len(lt.keys) + } + } + if err := problems.err(); err != nil { + return Report{}, err + } + for _, lt := range loaded { + if err := verify(ctx, tx, lt, problems); err != nil { + return Report{}, err + } + } + if gw := sources["gateway"]; gw != nil { + var err error + if report.UnindexedMessages, err = unindexedMessages(ctx, gw, loaded); err != nil { + return Report{}, err + } + } + return report, nil +} + +// hashSources fingerprints each source and its WAL, so a writer that was not +// stopped is caught rather than assumed away. +func hashSources(present []domain) (map[string][32]byte, error) { + out := map[string][32]byte{} + for _, d := range present { + for _, path := range []string{d.path, d.path + "-wal"} { + f, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) && path != d.path { + continue + } + if err != nil { + return nil, fmt.Errorf("legacy import: %w", err) + } + h := sha256.New() + _, err = io.Copy(h, f) + _ = f.Close() + if err != nil { + return nil, fmt.Errorf("legacy import: hash %s: %w", path, err) + } + out[path] = [32]byte(h.Sum(nil)) + } + } + return out, nil +} + +// checkSources finds the source rows Postgres would reject or reorder. +func checkSources(ctx context.Context, sources map[string]*legacyread.Source, problems *refusal) error { + if s := sources["state-store"]; s != nil { + for _, fk := range legacyread.IdentityForeignKeys { + orphans, err := s.Orphans(ctx, fk) + if err != nil { + return err + } + for _, o := range orphans { + problems.add("%s %s: %s %q has no row in %s (foreign key)", fk.Table, o.Key, fk.Column, o.Parent, fk.Parent) + } + } + aliases, err := s.DuplicateAliases(ctx) + if err != nil { + return err + } + for _, d := range aliases { + problems.add("identity_aliases %s: aliases collide on unique lower(alias) %q", strings.Join(d.Values, ", "), d.Key) + } + collisions, err := s.MicrosecondCollisions(ctx) + if err != nil { + return err + } + for _, c := range collisions { + problems.add("events %d and %d: at differs only below a microsecond (%s) while ids run the other way, so (at, id) order would change", + c.EarlierID, c.LaterID, c.At.Format("2006-01-02T15:04:05.000000Z")) + } + } + if s := sources["eda"]; s != nil { + dups, err := s.DuplicateBindings(ctx) + if err != nil { + return err + } + for _, d := range dups { + problems.add("bindings %s: more than one binding for source %q (unique source); resolve them and import again", + strings.Join(d.Values, ", "), d.Key) + } + } + return nil +} + +// checkTarget refuses a database that already completed an import, or that +// holds rows in a table owned by a domain being imported. The check is an +// allowlist of those tables, so goose's version table and other domains' +// tables never count. +func checkTarget(ctx context.Context, tx pgx.Tx, present []domain, problems *refusal) error { + done, err := postgresdb.New(tx).ImportCompleted(ctx) + if err != nil { + return fmt.Errorf("legacy import: completion record: %w", err) + } + if done { + problems.add("this database already holds a completed import") + return nil + } + for _, d := range present { + for _, t := range d.tables { + var rows bool + if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM "+pgx.Identifier{t.Name}.Sanitize()+")").Scan(&rows); err != nil { + return fmt.Errorf("legacy import: %s: %w", t.Name, err) + } + if rows { + problems.add("%s is not empty: the %s target already holds data", t.Name, d.name) + } + } + } + return nil +} diff --git a/internal/infrastructure/legacyimport/legacyimport_test.go b/internal/infrastructure/legacyimport/legacyimport_test.go new file mode 100644 index 000000000..0c07fb6cc --- /dev/null +++ b/internal/infrastructure/legacyimport/legacyimport_test.go @@ -0,0 +1,517 @@ +package legacyimport + +import ( + "bytes" + "crypto/sha256" + "database/sql" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + _ "modernc.org/sqlite" + + "github.com/samcharles93/archie-core/internal/infrastructure/postgres" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" +) + +func TestMain(m *testing.M) { os.Exit(pgtest.Main(m)) } + +// The fixtures apply the legacy schemas as their production openers leave +// them, columns added by ALTER TABLE included, with values in the layouts the +// production writers used. +const stateStoreDDL = ` +CREATE TABLE tasks (id INTEGER PRIMARY KEY AUTOINCREMENT, owner TEXT NOT NULL, repo TEXT NOT NULL, + issue_number INTEGER NOT NULL, title TEXT DEFAULT '', body TEXT DEFAULT '', labels TEXT DEFAULT '', + status TEXT DEFAULT 'queued', workflow TEXT DEFAULT '', stage TEXT DEFAULT '', branch TEXT DEFAULT '', + plan TEXT DEFAULT '', notes TEXT DEFAULT '', pr_number INTEGER DEFAULT 0, tokens_used INTEGER DEFAULT 0, + iterations INTEGER DEFAULT 0, attempt INTEGER DEFAULT 0, park_reason TEXT DEFAULT '', + watch_comment_id INTEGER DEFAULT 0, park_class TEXT DEFAULT 'needs_human', remediation_rounds INTEGER DEFAULT 0, + retry_count INTEGER DEFAULT 0, source TEXT DEFAULT 'forge', identity TEXT DEFAULT '', binding_id TEXT DEFAULT '', + binding_version INTEGER DEFAULT 0, review_payload TEXT DEFAULT '', workflow_definition_version INTEGER DEFAULT 0, + workflow_definition_digest TEXT DEFAULT '', workflow_definition_yaml TEXT DEFAULT '', + created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now'))); +ALTER TABLE tasks ADD COLUMN review_cursor INTEGER NOT NULL DEFAULT 0; +CREATE TABLE transitions (id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, + at TEXT DEFAULT (datetime('now')), from_status TEXT NOT NULL, to_status TEXT NOT NULL, detail TEXT DEFAULT ''); +CREATE TABLE events (id INTEGER PRIMARY KEY AUTOINCREMENT, at TEXT NOT NULL, kind TEXT NOT NULL, + task_id INTEGER DEFAULT 0, repo TEXT DEFAULT '', issue INTEGER DEFAULT 0, workflow TEXT DEFAULT '', + stage TEXT DEFAULT '', attempt INTEGER DEFAULT 0, actor_id TEXT DEFAULT '', actor_kind TEXT DEFAULT '', + principal_id TEXT DEFAULT '', detail TEXT DEFAULT '', data TEXT DEFAULT '{}'); +CREATE TABLE resources (kind TEXT PRIMARY KEY, value BLOB NOT NULL, version INTEGER NOT NULL, updated_at TEXT NOT NULL); +CREATE TABLE resource_history (id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, value BLOB NOT NULL, + version INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL, request_id TEXT NOT NULL, + expected_version INTEGER NOT NULL, current_version INTEGER NOT NULL, at TEXT NOT NULL); +CREATE TABLE channel_status (id TEXT PRIMARY KEY, name TEXT DEFAULT '', state TEXT DEFAULT '', detail TEXT DEFAULT '', + configured INTEGER NOT NULL DEFAULT 0, reload_supported INTEGER NOT NULL DEFAULT 0, observed_at TEXT NOT NULL); +CREATE TABLE apply_status (process TEXT NOT NULL, kind TEXT NOT NULL, applied_version INTEGER DEFAULT 0, + error TEXT DEFAULT '', reported_at TEXT NOT NULL, PRIMARY KEY (process, kind)); +CREATE TABLE config_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), schema TEXT DEFAULT '', document TEXT DEFAULT '', + published_at TEXT NOT NULL); +CREATE TABLE identities (id TEXT PRIMARY KEY, kind TEXT NOT NULL, display_name TEXT NOT NULL, lifecycle TEXT NOT NULL, + version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL); +CREATE TABLE identity_aliases (alias TEXT PRIMARY KEY COLLATE NOCASE, identity_id TEXT NOT NULL REFERENCES identities(id)); +CREATE TABLE identity_events (id INTEGER PRIMARY KEY AUTOINCREMENT, identity_id TEXT NOT NULL, event_type TEXT NOT NULL, + from_lifecycle TEXT NOT NULL, to_lifecycle TEXT NOT NULL, display_name TEXT NOT NULL, actor_id TEXT NOT NULL, + source TEXT NOT NULL, request_id TEXT NOT NULL UNIQUE, at TEXT NOT NULL); +CREATE TABLE identity_subjects (issuer TEXT NOT NULL, subject TEXT NOT NULL, + identity_id TEXT NOT NULL REFERENCES identities(id), bound_at TEXT NOT NULL, PRIMARY KEY (issuer, subject)); + +INSERT INTO tasks (id, owner, repo, issue_number, review_cursor, created_at, updated_at) VALUES + (1, 'o', 'r', 7, 3, '2026-01-01 00:00:00', '2026-01-01 00:00:01'), + (4, 'o', 'r', 1000000000000000, 0, '2026-01-02 00:00:00', '2026-01-02 00:00:00'), + (9, 'o', 'r', 8, 0, '2026-01-02 00:00:00', '2026-01-02 00:00:00'); +DELETE FROM tasks WHERE id = 9; +INSERT INTO transitions (task_id, at, from_status, to_status) VALUES (1, '2026-01-01 00:00:00', 'queued', 'running'); +INSERT INTO events (id, at, kind, data) VALUES + (1, '2026-01-01T00:00:00.000000100Z', 'a', 'null'), + (2, '2026-01-01T00:00:01.000000000Z', 'b', '{"k":1}'), + (5, '2026-01-01T00:00:00.500000000Z', 'c', '{}'); +INSERT INTO resources VALUES ('config', X'00ff10', 2, '2026-01-01T00:00:00.123456789Z'); +INSERT INTO resource_history (kind, value, version, actor, source, request_id, expected_version, current_version, at) + VALUES ('config', X'00ff10', 2, 'sam', 'ui', 'req-1', 1, 1, '2026-01-01T00:00:00Z'); +INSERT INTO channel_status VALUES ('tg', 'Telegram', 'up', '', 1, 0, '2026-01-01T00:00:00Z'); +INSERT INTO apply_status VALUES ('daemon', 'config', 2, '', '2026-01-01T00:00:00Z'); +INSERT INTO config_snapshot VALUES (1, 's', '{}', '2026-01-01T00:00:00Z'); +INSERT INTO identities VALUES ('id-1', 'agent', 'Archie', 'active', 1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z'); +INSERT INTO identity_aliases VALUES ('archie', 'id-1'); +INSERT INTO identity_events (identity_id, event_type, from_lifecycle, to_lifecycle, display_name, actor_id, source, request_id, at) + VALUES ('id-1', 'created', '', 'active', 'Archie', 'sam', 'boot', 'rq-1', '2026-01-01T00:00:00Z'); +INSERT INTO identity_subjects VALUES ('https://issuer', 'sub-1', 'id-1', '2026-01-01T00:00:00Z'); +` + +// secret is an encrypted binding secret envelope; it must arrive unchanged. +const secret = "arcie-binding:v1:0123456789abcdef:AbC-_dEf012" + +const edaDDL = ` +CREATE TABLE captures (id TEXT PRIMARY KEY NOT NULL, authenticated BOOLEAN DEFAULT FALSE NOT NULL, + body TEXT DEFAULT '' NOT NULL, content_type TEXT DEFAULT '' NOT NULL, headers JSON DEFAULT NULL, + received_at TEXT DEFAULT '' NOT NULL, remote_addr TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL); +CREATE TABLE mappings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, fields JSON DEFAULT NULL, + name TEXT DEFAULT '' NOT NULL, source_hint TEXT DEFAULT '' NOT NULL, updated_at TEXT DEFAULT '' NOT NULL); +CREATE TABLE bindings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, mapping TEXT DEFAULT '' NOT NULL, + name TEXT DEFAULT '' NOT NULL, owner TEXT DEFAULT '' NOT NULL, repo TEXT DEFAULT '' NOT NULL, + secret TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL, status TEXT DEFAULT '' NOT NULL, + updated_at TEXT DEFAULT '' NOT NULL, version NUMERIC DEFAULT 0 NOT NULL, workflow TEXT DEFAULT '' NOT NULL); +CREATE TABLE binding_dispatches (id TEXT PRIMARY KEY NOT NULL, binding TEXT DEFAULT '' NOT NULL, + binding_version NUMERIC DEFAULT 0 NOT NULL, capture TEXT DEFAULT '' NOT NULL, dispatched_at TEXT DEFAULT '' NOT NULL, + task_id NUMERIC DEFAULT 0 NOT NULL); +CREATE UNIQUE INDEX idx_binding_dispatch_once ON binding_dispatches (binding, capture); +CREATE TABLE playbook_dispatches (id TEXT PRIMARY KEY NOT NULL, action_id TEXT DEFAULT '' NOT NULL, + dispatched_at TEXT DEFAULT '' NOT NULL, event_id TEXT DEFAULT '' NOT NULL, playbook_id TEXT DEFAULT '' NOT NULL, + playbook_version TEXT DEFAULT '' NOT NULL); +CREATE TABLE tool_calls (id TEXT PRIMARY KEY NOT NULL, args JSON DEFAULT NULL, attempt NUMERIC DEFAULT 0 NOT NULL, + called_at TEXT DEFAULT '' NOT NULL, duration_ms NUMERIC DEFAULT 0 NOT NULL, error TEXT DEFAULT '' NOT NULL, + result JSON DEFAULT NULL, task_id NUMERIC DEFAULT 0 NOT NULL, tool TEXT DEFAULT '' NOT NULL); + +INSERT INTO captures VALUES ('rcap1', 1, '{}', 'application/json', '{"x":["1"]}', '2026-01-01T00:00:00.000000000Z', '127.0.0.1', 'gh'); +INSERT INTO captures VALUES ('rcap2', 0, '', '', NULL, '2026-01-01T00:00:01.000000000Z', '', 'gh'); +INSERT INTO mappings VALUES ('rmap1', '2026-01-01 00:00:00.000Z', '[]', 'm', 'gh', '2026-01-01 00:00:00.000Z'); +INSERT INTO bindings VALUES ('rbind1', '2026-01-01 00:00:00.000Z', 'rmap1', 'b', 'o', 'r', '` + secret + `', 'gh', 'armed', '2026-01-01 00:00:00.000Z', 1, 'implement'); +INSERT INTO binding_dispatches VALUES ('rdisp1', 'rbind1', 1, 'rcap1', '2026-01-01 00:00:00.000Z', 4); +INSERT INTO playbook_dispatches VALUES ('rpb1', 'act', '2026-01-01 00:00:00.000Z', 'ev', 'pb', 'v1'); +INSERT INTO tool_calls VALUES ('rtool1', NULL, 1, '2026-01-01 00:00:00.000Z', 12, '', '"exit 0"', 4, 'bash'); +` + +const gatewayDDL = ` +CREATE TABLE sessions (session_id TEXT PRIMARY KEY, platform TEXT NOT NULL DEFAULT '', bot_user TEXT NOT NULL DEFAULT '', + channel_id TEXT NOT NULL DEFAULT '', thread_id TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', + parent_session_id TEXT NOT NULL DEFAULT '', branch_name TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL DEFAULT 0, last_active_at INTEGER NOT NULL DEFAULT 0); +CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT NOT NULL, session_id TEXT NOT NULL, + source_id TEXT NOT NULL DEFAULT '', sender TEXT NOT NULL DEFAULT '', text TEXT NOT NULL DEFAULT '', ts INTEGER NOT NULL, + UNIQUE(session_id, message_id)); +ALTER TABLE messages ADD COLUMN role TEXT NOT NULL DEFAULT ''; +ALTER TABLE messages ADD COLUMN sender_id TEXT NOT NULL DEFAULT ''; +CREATE VIRTUAL TABLE messages_fts USING fts5(sender, text, content='messages', content_rowid='id'); +CREATE TRIGGER messages_ai AFTER INSERT ON messages BEGIN + INSERT INTO messages_fts(rowid, sender, text) VALUES (new.id, new.sender, new.text); +END; +CREATE TABLE turns (turn_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, source_id TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL, attempt INTEGER NOT NULL DEFAULT 0, owner_id TEXT NOT NULL DEFAULT '', + input_message_id TEXT NOT NULL DEFAULT '', assistant_message_id TEXT NOT NULL DEFAULT '', + partial_text TEXT NOT NULL DEFAULT '', response_text TEXT NOT NULL DEFAULT '', tool_calls TEXT NOT NULL DEFAULT '[]', + error TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL); + +INSERT INTO sessions VALUES ('s1', 'telegram', 'bot', 'c1', '', 'Chat', '', '', 1000, 2000); +INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m1', 's1', 'sam', 'hello there', 1000, 'user'); +INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m2', 's1', 'archie', 'hi', 1001, 'assistant'); +INSERT INTO turns VALUES ('t1', 's1', '', 'done', 1, '', 'm1', 'm2', '', 'hi', '[]', '', 1000, 1001); +` + +// install writes the three legacy files the way an existing install holds +// them and returns their paths. mutate runs extra statements against a file +// before the import sees it. +func install(t *testing.T, mutate map[string]string) Sources { + t.Helper() + dir := t.TempDir() + src := Sources{ + StateStore: filepath.Join(dir, "archie.db-tasks.sqlite"), + EDA: filepath.Join(dir, "archie.db-eda.sqlite"), + Gateway: filepath.Join(dir, "archie.db-conversations.sqlite"), + } + for path, ddl := range map[string]string{src.StateStore: stateStoreDDL, src.EDA: edaDDL, src.Gateway: gatewayDDL} { + sqliteExec(t, path, ddl) + } + for path, stmt := range mutate { + switch path { + case "state": + sqliteExec(t, src.StateStore, stmt) + case "eda": + sqliteExec(t, src.EDA, stmt) + case "gateway": + sqliteExec(t, src.Gateway, stmt) + } + } + return src +} + +func sqliteExec(t *testing.T, path, stmt string) { + t.Helper() + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer func() { _ = db.Close() }() + if _, err := db.ExecContext(t.Context(), stmt); err != nil { + t.Fatalf("fixture %s: %v", filepath.Base(path), err) + } +} + +func hashFiles(t *testing.T, dir string) map[string][32]byte { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read dir: %v", err) + } + out := map[string][32]byte{} + for _, e := range entries { + b, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + t.Fatalf("read %s: %v", e.Name(), err) + } + out[e.Name()] = sha256.Sum256(b) + } + return out +} + +func target(t *testing.T) *pgxpool.Pool { + t.Helper() + pool, err := postgres.Open(t.Context(), pgtest.URL(t)) + if err != nil { + t.Fatalf("postgres.Open: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func migrated(t *testing.T) *pgxpool.Pool { + t.Helper() + pool := target(t) + if err := postgres.Migrate(t.Context(), pool, postgres.Migrations()); err != nil { + t.Fatalf("Migrate: %v", err) + } + return pool +} + +func count(t *testing.T, pool *pgxpool.Pool, table string) int { + t.Helper() + var n int + if err := pool.QueryRow(t.Context(), "SELECT count(*) FROM "+table).Scan(&n); err != nil { + t.Fatalf("count %s: %v", table, err) + } + return n +} + +func TestImportCopiesEverySourceAndLeavesThemUntouched(t *testing.T) { + src := install(t, nil) + before := hashFiles(t, filepath.Dir(src.StateStore)) + pool := target(t) + + report, err := Import(t.Context(), pool, src) + if err != nil { + t.Fatalf("Import: %v", err) + } + + if after := hashFiles(t, filepath.Dir(src.StateStore)); len(after) != len(before) { + t.Fatalf("source directory changed: %d files before, %d after", len(before), len(after)) + } else { + for name, sum := range before { + if after[name] != sum { + t.Errorf("source %s changed during import", name) + } + } + } + + done, err := postgres.ImportComplete(t.Context(), pool) + if err != nil || !done { + t.Fatalf("ImportComplete = %v, %v; want true", done, err) + } + + counts := map[string]int{ + "tasks": 2, "transitions": 1, "events": 3, "resources": 1, "resource_history": 1, + "channel_status": 1, "apply_status": 1, "config_snapshot": 1, "identities": 1, + "identity_aliases": 1, "identity_events": 1, "identity_subjects": 1, + "captures": 2, "mappings": 1, "bindings": 1, "binding_dispatches": 1, "playbook_dispatches": 1, + "tool_calls": 1, "sessions": 1, "messages": 2, "turns": 1, + } + for table, want := range counts { + if got := count(t, pool, table); got != want { + t.Errorf("%s rows = %d, want %d", table, got, want) + } + if got := report.Rows[table]; got != want { + t.Errorf("report rows[%s] = %d, want %d", table, got, want) + } + } + + ctx := t.Context() + var ( + reviewCursor int64 + value []byte + gotSecret string + headers string + result string + configured bool + ) + checks := []struct { + query string + dest any + }{ + {"SELECT review_cursor FROM tasks WHERE id = 1", &reviewCursor}, + {"SELECT value FROM resource_history WHERE id = 1", &value}, + {"SELECT secret FROM bindings WHERE id = 'rbind1'", &gotSecret}, + {"SELECT headers FROM captures WHERE id = 'rcap2'", &headers}, + {"SELECT result FROM tool_calls WHERE id = 'rtool1'", &result}, + {"SELECT configured FROM channel_status WHERE id = 'tg'", &configured}, + } + for _, c := range checks { + if err := pool.QueryRow(ctx, c.query).Scan(c.dest); err != nil { + t.Fatalf("%s: %v", c.query, err) + } + } + if reviewCursor != 3 { + t.Errorf("review_cursor = %d, want 3", reviewCursor) + } + if !bytes.Equal(value, []byte{0x00, 0xff, 0x10}) { + t.Errorf("resource_history.value = %x, want 00ff10", value) + } + if gotSecret != secret { + t.Errorf("binding secret = %q, want %q", gotSecret, secret) + } + if headers != "" || result != "exit 0" || !configured { + t.Errorf("headers %q, result %q, configured %v; want \"\", \"exit 0\", true", headers, result, configured) + } + + // Event order is (at, id) as the source had it, and the next event id + // follows AUTOINCREMENT's high-water mark rather than reusing one. + rows, err := pool.Query(ctx, "SELECT id FROM events ORDER BY at, id") + if err != nil { + t.Fatalf("events: %v", err) + } + var order []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + t.Fatalf("scan: %v", err) + } + order = append(order, id) + } + if want := []int64{1, 5, 2}; !equalInts(order, want) { + t.Errorf("event order = %v, want %v", order, want) + } + nextIDs := map[string]int64{"tasks": 10, "events": 6, "messages": 3} + for table, want := range nextIDs { + var got int64 + if err := pool.QueryRow(ctx, "SELECT nextval(pg_get_serial_sequence($1, 'id'))", table).Scan(&got); err != nil { + t.Fatalf("nextval %s: %v", table, err) + } + if got != want { + t.Errorf("next %s id = %d, want %d", table, got, want) + } + } + + var hits int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM messages WHERE search @@ to_tsquery('simple', 'hello')").Scan(&hits); err != nil { + t.Fatalf("search: %v", err) + } + if hits != 1 { + t.Errorf("search hits for an imported message = %d, want 1", hits) + } +} + +func equalInts(a, b []int64) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestImportRefuses(t *testing.T) { + tests := []struct { + name string + mutate map[string]string + seed func(t *testing.T, pool *pgxpool.Pool) + want []string + is error + }{ + { + name: "non-empty target", + seed: func(t *testing.T, pool *pgxpool.Pool) { + mustExec(t, pool, "INSERT INTO transitions (task_id, from_status, to_status) VALUES (1, 'a', 'b')") + }, + want: []string{"transitions", "not empty"}, + }, + { + name: "completed import", + seed: func(t *testing.T, pool *pgxpool.Pool) { + mustExec(t, pool, "INSERT INTO import_completion (id, sources, report) VALUES (1, 's', 'r')") + }, + want: []string{"already"}, + }, + { + name: "live state store", + seed: func(t *testing.T, pool *pgxpool.Pool) { holdOwnership(t, pool, postgres.OwnerStateStore) }, + want: []string{"state-store"}, + is: postgres.ErrOwned, + }, + { + name: "live gateway", + seed: func(t *testing.T, pool *pgxpool.Pool) { holdOwnership(t, pool, postgres.OwnerGateway) }, + want: []string{"gateway"}, + is: postgres.ErrOwned, + }, + { + name: "orphaned alias", + mutate: map[string]string{"state": "INSERT INTO identity_aliases VALUES ('ghost', 'id-404')"}, + want: []string{"identity_aliases", "ghost", "id-404"}, + }, + { + name: "duplicate bindings", + mutate: map[string]string{"eda": "INSERT INTO bindings VALUES ('rbind2', '2026-01-01 00:00:00.000Z', '', 'b2', 'o', 'r', '', 'gh', 'pending_approval', '2026-01-01 00:00:00.000Z', 1, 'implement')"}, + want: []string{"bindings", "gh", "rbind1", "rbind2"}, + }, + { + name: "microsecond collision", + mutate: map[string]string{"state": "INSERT INTO events (id, at, kind) VALUES (7, '2026-01-01T00:00:00.000000050Z', 'd')"}, + want: []string{"events", "7", "microsecond"}, + }, + { + name: "unparseable timestamp", + mutate: map[string]string{"eda": "UPDATE captures SET received_at = 'yesterday' WHERE id = 'rcap2'"}, + want: []string{"captures", "rcap2", "yesterday", "received_at"}, + }, + { + name: "null in a required column", + mutate: map[string]string{"state": "UPDATE tasks SET title = NULL WHERE id = 4"}, + want: []string{"tasks", "4", "title", "NULL"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + src := install(t, tt.mutate) + pool := migrated(t) + if tt.seed != nil { + tt.seed(t, pool) + } + _, err := Import(t.Context(), pool, src) + if err == nil { + t.Fatal("Import succeeded, want a refusal") + } + if tt.is != nil && !errors.Is(err, tt.is) { + t.Errorf("Import error %v, want %v", err, tt.is) + } + for _, w := range tt.want { + if !strings.Contains(err.Error(), w) { + t.Errorf("refusal %q does not name %q", err, w) + } + } + // Nothing of the import survives a refusal: the store stays + // unserveable rather than half-imported. + if got := count(t, pool, "tasks"); got != 0 { + t.Errorf("tasks rows after refusal = %d, want 0", got) + } + if got := count(t, pool, "import_completion"); tt.name != "completed import" && got != 0 { + t.Errorf("completion records after refusal = %d, want 0", got) + } + }) + } +} + +func mustExec(t *testing.T, pool *pgxpool.Pool, stmt string) { + t.Helper() + if _, err := pool.Exec(t.Context(), stmt); err != nil { + t.Fatalf("%s: %v", stmt, err) + } +} + +// holdOwnership claims name from a separate pool, as a live service would. +func holdOwnership(t *testing.T, pool *pgxpool.Pool, name string) { + t.Helper() + other, err := pgxpool.New(t.Context(), pool.Config().ConnString()) + if err != nil { + t.Fatalf("second pool: %v", err) + } + t.Cleanup(other.Close) + claim, err := postgres.AcquireOwnership(t.Context(), other, name) + if err != nil { + t.Fatalf("claim %s: %v", name, err) + } + t.Cleanup(func() { _ = claim.Release(t.Context()) }) +} + +// TestSiblingDomainDoesNotBlock pins that "non-empty" is per domain: a +// domain with no legacy source is not imported, so rows in its tables do not +// refuse the others. +func TestSiblingDomainDoesNotBlock(t *testing.T) { + src := install(t, nil) + src.Gateway = filepath.Join(t.TempDir(), "absent.sqlite") + pool := migrated(t) + mustExec(t, pool, "INSERT INTO sessions (session_id) VALUES ('live')") + if _, err := Import(t.Context(), pool, src); err != nil { + t.Fatalf("Import: %v", err) + } + if got := count(t, pool, "tasks"); got != 2 { + t.Fatalf("tasks rows = %d, want 2", got) + } +} + +func TestImportRefusesWithNoSources(t *testing.T) { + dir := t.TempDir() + src := Sources{StateStore: filepath.Join(dir, "a"), EDA: filepath.Join(dir, "b"), Gateway: filepath.Join(dir, "c")} + if _, err := Import(t.Context(), migrated(t), src); err == nil || !strings.Contains(err.Error(), "no legacy") { + t.Fatalf("Import = %v, want a refusal naming no legacy sources", err) + } +} + +func TestHasLegacyData(t *testing.T) { + dir := t.TempDir() + absent := Sources{StateStore: filepath.Join(dir, "a"), EDA: filepath.Join(dir, "b"), Gateway: filepath.Join(dir, "c")} + onlyGateway := absent + onlyGateway.Gateway = install(t, nil).Gateway + tests := []struct { + name string + src Sources + want bool + }{ + {"fresh install", absent, false}, + {"one source with rows", onlyGateway, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := HasLegacyData(t.Context(), tt.src) + if err != nil { + t.Fatalf("HasLegacyData: %v", err) + } + if got != tt.want { + t.Fatalf("HasLegacyData = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/infrastructure/legacyimport/load.go b/internal/infrastructure/legacyimport/load.go new file mode 100644 index 000000000..4d5096601 --- /dev/null +++ b/internal/infrastructure/legacyimport/load.go @@ -0,0 +1,288 @@ +package legacyimport + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "slices" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/samcharles93/archie-core/internal/infrastructure/legacyread" +) + +// droppedColumns are legacy columns with no target column by design: +// PocketBase gave every ledger row a random record id, and the Postgres +// ledgers are keyed by their natural unique key instead. +var droppedColumns = map[string]bool{ + "binding_dispatches.id": true, + "playbook_dispatches.id": true, +} + +// targetKeys override a table's legacy key where the target has no column +// for it. +var targetKeys = map[string][]string{ + "binding_dispatches": {"binding", "capture"}, + "playbook_dispatches": {"playbook_id", "playbook_version", "event_id", "action_id"}, +} + +// jsonColumns are PocketBase JSON fields, which may be NULL in the source and +// map to a NOT NULL text column defaulting to the empty string. unwrap marks the fields the legacy reader +// decoded from a JSON string (jsonTextToString) rather than returning raw. +var jsonColumns = map[string]struct{ unwrap bool }{ + "captures.headers": {}, + "mappings.fields": {}, + "tool_calls.args": {unwrap: true}, + "tool_calls.result": {unwrap: true}, +} + +// timeLayouts are every layout a legacy timestamp was written in: the +// fixed-width cursor layout, Go's RFC 3339 forms, PocketBase's autodate +// layout and SQLite's datetime('now'). +var timeLayouts = []string{ + "2006-01-02T15:04:05.000000000Z", + time.RFC3339Nano, + time.RFC3339, + "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02 15:04:05.999999999", +} + +type column struct { + name string + dataType string + identity bool + source int // index into the legacy row +} + +// loadedTable is what load wrote, for verify to check. +type loadedTable struct { + table legacyread.Table + rows legacyread.Rows + keys []string // target keys of the rows written, in source read order + keyCols []string + identity bool +} + +// load reads t from src, converts every row to the target's types and copies +// them in with their historical ids. A row it cannot convert is added to +// problems and nothing of the table is written. +func load(ctx context.Context, tx pgx.Tx, src *legacyread.Source, t legacyread.Table, problems *refusal) (loadedTable, error) { + cols, err := plan(ctx, tx, t, problems) + if err != nil || cols == nil { + return loadedTable{table: t}, err + } + rows, err := src.Read(ctx, t) + if err != nil { + return loadedTable{}, err + } + lt := loadedTable{table: t, rows: rows, keyCols: t.Key} + if k, ok := targetKeys[t.Name]; ok { + lt.keyCols = k + } + names := make([]string, len(cols)) + for i, c := range cols { + names[i] = c.name + lt.identity = lt.identity || c.identity + } + out, ok := convertRows(lt, cols, names, problems) + if !ok { + return lt, nil + } + for _, row := range out { + key := make([]string, len(lt.keyCols)) + for j, k := range lt.keyCols { + key[j] = fmt.Sprint(row[slices.Index(names, k)]) + } + lt.keys = append(lt.keys, strings.Join(key, "/")) + } + if len(out) > 0 { + if _, err := tx.CopyFrom(ctx, pgx.Identifier{t.Name}, names, pgx.CopyFromRows(out)); err != nil { + problems.add("%s: the target rejected the rows: %v", t.Name, err) + return lt, nil + } + } + if lt.identity { + return lt, advanceSequence(ctx, tx, src, t.Name) + } + return lt, nil +} + +// convertRows converts every row of lt, adding each value it cannot convert +// to problems. +func convertRows(lt loadedTable, cols []column, names []string, problems *refusal) ([][]any, bool) { + sourceKeys := lt.rows.Keys() + out := make([][]any, 0, len(lt.rows.Rows)) + ok := true + for i, row := range lt.rows.Rows { + converted := make([]any, len(cols)) + for j, c := range cols { + v, err := convert(lt.table.Name+"."+c.name, c.dataType, row[c.source]) + if err != nil { + problems.add("%s %s: column %s value %s: %v", lt.table.Name, sourceKeys[i], names[j], quote(row[c.source]), err) + ok = false + } + converted[j] = v + } + out = append(out, converted) + } + return out, ok +} + +// advanceSequence sets table's identity sequence to hand out the id the +// legacy AUTOINCREMENT would have, even for a table whose rows were all +// deleted. is_called = false makes that id the next one, which keeps +// AUTOINCREMENT's promise that a deleted top id is never reused. +func advanceSequence(ctx context.Context, tx pgx.Tx, src *legacyread.Source, table string) error { + seq, err := src.SequenceOf(ctx, table) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, "SELECT setval(pg_get_serial_sequence($1, 'id'), $2, false)", table, seq.Next); err != nil { + return fmt.Errorf("legacy import: advance %s sequence: %w", table, err) + } + return nil +} + +// plan matches the target table's columns to the legacy ones. A column on +// either side with no counterpart is a schema gap the import refuses to +// guess about; it returns nil columns then. +func plan(ctx context.Context, tx pgx.Tx, t legacyread.Table, problems *refusal) ([]column, error) { + rows, err := tx.Query(ctx, `SELECT column_name, data_type, is_identity = 'YES' + FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1 AND is_generated = 'NEVER' + ORDER BY ordinal_position`, t.Name) + if err != nil { + return nil, fmt.Errorf("legacy import: %s columns: %w", t.Name, err) + } + cols, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (column, error) { + var c column + err := row.Scan(&c.name, &c.dataType, &c.identity) + return c, err + }) + if err != nil { + return nil, fmt.Errorf("legacy import: %s columns: %w", t.Name, err) + } + gap := len(cols) == 0 + if gap { + problems.add("%s: no target table", t.Name) + } + targetNames := map[string]bool{} + for i := range cols { + targetNames[cols[i].name] = true + cols[i].source = slices.Index(t.Columns, cols[i].name) + if cols[i].source < 0 { + problems.add("%s.%s: the target column has no legacy column", t.Name, cols[i].name) + gap = true + } + } + for _, c := range t.Columns { + if !targetNames[c] && !droppedColumns[t.Name+"."+c] { + problems.add("%s.%s: the legacy column has no target column", t.Name, c) + gap = true + } + } + if gap { + return nil, nil + } + return cols, nil +} + +// convert maps one SQLite value to the Go value pgx writes to a column of +// dataType. It never coerces: a value the column cannot hold is an error +// naming the rule it breaks. +func convert(qualified, dataType string, v any) (any, error) { + if j, ok := jsonColumns[qualified]; ok { + return convertJSON(v, j.unwrap) + } + if v == nil { + return nil, fmt.Errorf("NULL in a NOT NULL %s column", dataType) + } + conv, ok := converters[dataType] + if !ok { + return nil, fmt.Errorf("unsupported target type %s", dataType) + } + return conv(v) +} + +func convertJSON(v any, unwrap bool) (any, error) { + switch s := v.(type) { + case nil: + return "", nil + case string: + if unwrap { + var decoded string + if json.Unmarshal([]byte(s), &decoded) == nil { + return decoded, nil + } + } + return s, nil + } + return nil, errors.New("text: not a string") +} + +// converters are keyed by information_schema data_type. +var converters = map[string]func(any) (any, error){ + "text": func(v any) (any, error) { + if s, ok := v.(string); ok { + return s, nil + } + return nil, errors.New("text: not a string") + }, + "bigint": func(v any) (any, error) { + switch n := v.(type) { + case int64: + return n, nil + case float64: + if n == math.Trunc(n) && math.Abs(n) < 1<<63 { + return int64(n), nil + } + } + return nil, errors.New("bigint: not an integer") + }, + "boolean": func(v any) (any, error) { + switch b := v.(type) { + case bool: + return b, nil + case int64: + if b == 0 || b == 1 { + return b == 1, nil + } + } + return nil, errors.New("boolean: not 0 or 1") + }, + "bytea": func(v any) (any, error) { + switch b := v.(type) { + case []byte: + return b, nil + case string: + return []byte(b), nil + } + return nil, errors.New("bytea: not a blob") + }, + "timestamp with time zone": func(v any) (any, error) { + if s, ok := v.(string); ok { + for _, layout := range timeLayouts { + if at, err := time.Parse(layout, s); err == nil { + return at.UTC(), nil + } + } + } + return nil, errors.New("timestamptz: not a timestamp in any legacy layout") + }, +} + +func quote(v any) string { + switch s := v.(type) { + case nil: + return "NULL" + case string: + return fmt.Sprintf("%q", s) + case []byte: + return fmt.Sprintf("x'%x'", s) + } + return fmt.Sprint(v) +} diff --git a/internal/infrastructure/legacyimport/verify.go b/internal/infrastructure/legacyimport/verify.go new file mode 100644 index 000000000..6fa4e9bde --- /dev/null +++ b/internal/infrastructure/legacyimport/verify.go @@ -0,0 +1,171 @@ +package legacyimport + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/jackc/pgx/v5" + + "github.com/samcharles93/archie-core/internal/infrastructure/legacyread" +) + +// blobColumns are the columns whose bytes must survive exactly: resource +// documents, and binding secrets, which stay decryptable by the configured +// keyring only if the envelope is untouched. +var blobColumns = map[string]string{ + "resources": "value", + "resource_history": "value", + "bindings": "secret", +} + +// verify reads back what load wrote for lt and compares it with the source: +// the key set (a count passes on a swallowed row when ids have gaps), the +// (at, id) event order, blob bytes and the identity sequence. +func verify(ctx context.Context, tx pgx.Tx, lt loadedTable, problems *refusal) error { + name := lt.table.Name + ident := pgx.Identifier{name}.Sanitize() + keyExprs := make([]string, len(lt.keyCols)) + for i, k := range lt.keyCols { + keyExprs[i] = pgx.Identifier{k}.Sanitize() + "::text" + } + keyExpr := "concat_ws('/', " + strings.Join(keyExprs, ", ") + ")" + + targetKeys, err := strs(ctx, tx, "SELECT "+keyExpr+" FROM "+ident) + if err != nil { + return fmt.Errorf("legacy import: verify %s: %w", name, err) + } + if len(targetKeys) != len(lt.keys) { + problems.add("%s: %d rows read, %d rows in the target", name, len(lt.keys), len(targetKeys)) + } + if d := legacyread.CompareKeys(lt.keys, targetKeys); len(d.Missing)+len(d.Extra) > 0 { + problems.add("%s: rows missing from the target %v, rows only in the target %v", name, d.Missing, d.Extra) + } + + if name == "events" { + if err := verifyEventOrder(ctx, tx, lt, problems); err != nil { + return err + } + } + if col, ok := blobColumns[name]; ok { + if err := verifyBlobs(ctx, tx, lt, col, keyExpr, problems); err != nil { + return err + } + } + if lt.identity { + return verifySequence(ctx, tx, name, problems) + } + return nil +} + +// verifySequence asserts the next id the sequence hands out is above every +// imported id, reading its state rather than calling nextval. +func verifySequence(ctx context.Context, tx pgx.Tx, name string, problems *refusal) error { + var seq string + var maxID int64 + if err := tx.QueryRow(ctx, "SELECT pg_get_serial_sequence($1, 'id'), (SELECT COALESCE(max(id), 0) FROM "+ + pgx.Identifier{name}.Sanitize()+")", name).Scan(&seq, &maxID); err != nil { + return fmt.Errorf("legacy import: verify %s sequence: %w", name, err) + } + var ( + last int64 + called bool + ) + // seq comes from pg_get_serial_sequence, which returns it quoted. + if err := tx.QueryRow(ctx, "SELECT last_value, is_called FROM "+seq).Scan(&last, &called); err != nil { + return fmt.Errorf("legacy import: verify %s sequence: %w", name, err) + } + next := last + if called { + next++ + } + if next <= maxID { + problems.add("%s: next id %d would reuse an imported id (max %d)", name, next, maxID) + } + return nil +} + +// verifyEventOrder compares the ids in (at, id) order on both sides. The +// source was read in that order; a difference means microsecond storage +// reordered history, and a cursor reader would skip a row. +func verifyEventOrder(ctx context.Context, tx pgx.Tx, lt loadedTable, problems *refusal) error { + source := make([]int64, len(lt.keys)) + for i, k := range lt.keys { + id, err := strconv.ParseInt(k, 10, 64) + if err != nil { + return fmt.Errorf("legacy import: event key %q: %w", k, err) + } + source[i] = id + } + rows, err := tx.Query(ctx, "SELECT id FROM events ORDER BY at, id") + if err != nil { + return fmt.Errorf("legacy import: verify events order: %w", err) + } + target, err := pgx.CollectRows(rows, pgx.RowTo[int64]) + if err != nil { + return fmt.Errorf("legacy import: verify events order: %w", err) + } + if d := legacyread.CompareOrder(source, target); d != nil { + problems.add("events: (at, id) order differs at position %d: source id %d, target id %d", d.Position, d.Source, d.Target) + } + return nil +} + +func verifyBlobs(ctx context.Context, tx pgx.Tx, lt loadedTable, col, keyExpr string, problems *refusal) error { + source := map[string][]byte{} + for i, k := range lt.keys { + switch v := lt.rows.Value(i, col).(type) { + case []byte: + source[k] = v + case string: + source[k] = []byte(v) + } + } + rows, err := tx.Query(ctx, fmt.Sprintf("SELECT %s, %s FROM %s", keyExpr, + pgx.Identifier{col}.Sanitize(), pgx.Identifier{lt.table.Name}.Sanitize())) + if err != nil { + return fmt.Errorf("legacy import: verify %s.%s: %w", lt.table.Name, col, err) + } + target := map[string][]byte{} + for rows.Next() { + var ( + k string + v []byte + ) + if err := rows.Scan(&k, &v); err != nil { + rows.Close() + return fmt.Errorf("legacy import: verify %s.%s: %w", lt.table.Name, col, err) + } + target[k] = v + } + if err := rows.Err(); err != nil { + return fmt.Errorf("legacy import: verify %s.%s: %w", lt.table.Name, col, err) + } + if diff := legacyread.CompareBlobs(source, target); len(diff) > 0 { + problems.add("%s.%s: bytes differ from the source for %v", lt.table.Name, col, diff) + } + return nil +} + +// unindexedMessages returns the source message ids the FTS5 index missed. +func unindexedMessages(ctx context.Context, gw *legacyread.Source, loaded []loadedTable) ([]string, error) { + indexed, err := gw.IndexedMessageIDs(ctx) + if err != nil { + return nil, err + } + for _, lt := range loaded { + if lt.table.Name == "messages" { + return legacyread.CompareKeys(indexed, lt.keys).Extra, nil + } + } + return nil, nil +} + +func strs(ctx context.Context, tx pgx.Tx, query string) ([]string, error) { + rows, err := tx.Query(ctx, query) + if err != nil { + return nil, err + } + return pgx.CollectRows(rows, pgx.RowTo[string]) +} diff --git a/internal/infrastructure/legacyread/presence.go b/internal/infrastructure/legacyread/presence.go new file mode 100644 index 000000000..5697c8f5f --- /dev/null +++ b/internal/infrastructure/legacyread/presence.go @@ -0,0 +1,55 @@ +package legacyread + +import ( + "context" + "errors" + "fmt" + "os" +) + +// HasData reports whether the legacy file at path holds any row in tables. A +// missing file, and a table the file never created, hold nothing. It is the +// legacy half of the boot gate: a Postgres database without a completed import +// must not serve while this is true for any source. +func HasData(ctx context.Context, path string, tables []Table) (bool, error) { + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return false, nil + } + src, err := Open(ctx, path) + if err != nil { + return false, err + } + defer func() { _ = src.Close() }() + for _, t := range tables { + var present bool + if err := src.db.QueryRowContext(ctx, + "SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)", t.Name).Scan(&present); err != nil { + return false, fmt.Errorf("legacyread: %s: %w", path, err) + } + if !present { + continue + } + var rows bool + if err := src.db.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM "+t.Name+")").Scan(&rows); err != nil { + return false, fmt.Errorf("legacyread: %s: %s: %w", path, t.Name, err) + } + if rows { + return true, nil + } + } + return false, nil +} + +// SequenceOf returns the state of table's AUTOINCREMENT id column. +func (s *Source) SequenceOf(ctx context.Context, table string) (Sequence, error) { + seq := Sequence{Table: table} + var high int64 + err := s.db.QueryRowContext(ctx, fmt.Sprintf( + "SELECT COALESCE((SELECT max(id) FROM %[1]s), 0), COALESCE((SELECT seq FROM sqlite_sequence WHERE name = '%[1]s'), 0)", + table)).Scan(&seq.MaxID, &high) + if err != nil { + return Sequence{}, fmt.Errorf("legacyread: sequence %s: %w", table, err) + } + seq.Next = max(seq.MaxID, high) + 1 + return seq, nil +} diff --git a/internal/infrastructure/legacyread/presence_test.go b/internal/infrastructure/legacyread/presence_test.go new file mode 100644 index 000000000..ccc2018c1 --- /dev/null +++ b/internal/infrastructure/legacyread/presence_test.go @@ -0,0 +1,46 @@ +package legacyread + +import ( + "path/filepath" + "testing" +) + +func TestHasData(t *testing.T) { + emptyGateway := `CREATE TABLE sessions (session_id TEXT PRIMARY KEY); +CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT);` + tests := []struct { + name string + path func(t *testing.T) string + tables []Table + want bool + }{ + {"missing file", func(t *testing.T) string { return filepath.Join(t.TempDir(), "absent.sqlite") }, GatewayTables, false}, + {"tables without rows", func(t *testing.T) string { return fixture(t, "gw.sqlite", emptyGateway) }, GatewayTables, false}, + {"rows present", func(t *testing.T) string { return fixture(t, "gw.sqlite", gatewayDDL) }, GatewayTables, true}, + {"rows in the state store", func(t *testing.T) string { return fixture(t, "archie.db", stateStoreDDL) }, StateStoreTables, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := HasData(t.Context(), tt.path(t), tt.tables) + if err != nil { + t.Fatalf("HasData: %v", err) + } + if got != tt.want { + t.Fatalf("HasData = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSequenceOfMessages(t *testing.T) { + path := fixture(t, "gw.sqlite", gatewayDDL) + exec(t, path, `DELETE FROM messages WHERE message_id = 'm2'`) + src := openSource(t, path) + got, err := src.SequenceOf(t.Context(), "messages") + if err != nil { + t.Fatalf("SequenceOf: %v", err) + } + if want := (Sequence{Table: "messages", MaxID: 1, Next: 3}); got != want { + t.Fatalf("SequenceOf(messages) = %+v, want %+v", got, want) + } +} diff --git a/internal/infrastructure/legacyread/verify.go b/internal/infrastructure/legacyread/verify.go index d3ecc1435..6fb2f4d68 100644 --- a/internal/infrastructure/legacyread/verify.go +++ b/internal/infrastructure/legacyread/verify.go @@ -242,15 +242,10 @@ var sequenceTables = []string{"tasks", "transitions", "events", "resource_histor func (s *Source) Sequences(ctx context.Context) ([]Sequence, error) { out := make([]Sequence, 0, len(sequenceTables)) for _, table := range sequenceTables { - seq := Sequence{Table: table} - var high int64 - err := s.db.QueryRowContext(ctx, fmt.Sprintf( - "SELECT COALESCE((SELECT max(id) FROM %[1]s), 0), COALESCE((SELECT seq FROM sqlite_sequence WHERE name = '%[1]s'), 0)", - table)).Scan(&seq.MaxID, &high) + seq, err := s.SequenceOf(ctx, table) if err != nil { - return nil, fmt.Errorf("legacyread: sequence %s: %w", table, err) + return nil, err } - seq.Next = max(seq.MaxID, high) + 1 out = append(out, seq) } return out, nil diff --git a/internal/infrastructure/postgres/import_completion.go b/internal/infrastructure/postgres/import_completion.go new file mode 100644 index 000000000..32071fd71 --- /dev/null +++ b/internal/infrastructure/postgres/import_completion.go @@ -0,0 +1,29 @@ +package postgres + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb" +) + +// Service owner names: every Archie process that serves this database holds +// one of these claims for its whole life, so an offline rewrite checks them +// all. +const ( + OwnerStateStore = "state-store" + OwnerGateway = "gateway" +) + +// ImportComplete reports whether the one-time legacy import committed and +// verified against this migrated database. Serving over non-empty legacy +// files is refused without it. +func ImportComplete(ctx context.Context, pool *pgxpool.Pool) (bool, error) { + done, err := postgresdb.New(pool).ImportCompleted(ctx) + if err != nil { + return false, fmt.Errorf("postgres: import completion: %w", err) + } + return done, nil +} diff --git a/internal/infrastructure/postgres/import_completion_test.go b/internal/infrastructure/postgres/import_completion_test.go new file mode 100644 index 000000000..cd6e459df --- /dev/null +++ b/internal/infrastructure/postgres/import_completion_test.go @@ -0,0 +1,39 @@ +package postgres + +import ( + "testing" + + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" +) + +func TestImportComplete(t *testing.T) { + tests := []struct { + name string + record bool + want bool + }{ + {name: "migrated database without a record", want: false}, + {name: "committed record", record: true, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pool := openPool(t, pgtest.URL(t)) + if err := Migrate(t.Context(), pool, Migrations()); err != nil { + t.Fatalf("Migrate: %v", err) + } + if tt.record { + if _, err := pool.Exec(t.Context(), + "INSERT INTO import_completion (id, sources, report) VALUES (1, 's', 'r')"); err != nil { + t.Fatalf("seed record: %v", err) + } + } + got, err := ImportComplete(t.Context(), pool) + if err != nil { + t.Fatalf("ImportComplete: %v", err) + } + if got != tt.want { + t.Fatalf("ImportComplete = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/infrastructure/postgres/migrations/0005_import_completion.sql b/internal/infrastructure/postgres/migrations/0005_import_completion.sql new file mode 100644 index 000000000..f44a91023 --- /dev/null +++ b/internal/infrastructure/postgres/migrations/0005_import_completion.sql @@ -0,0 +1,19 @@ +-- import_completion records that the one-time import of an existing install's +-- SQLite stores committed and passed verification. It is written in the same +-- transaction as the imported rows, so a record exists exactly when the data +-- does. An empty database is not evidence that an import succeeded, only that +-- nothing has written to it yet, so serving over non-empty legacy files +-- requires this row. + +-- +goose Up + +CREATE TABLE import_completion ( + id bigint PRIMARY KEY CHECK (id = 1), + completed_at timestamptz NOT NULL DEFAULT now(), + sources text NOT NULL, + report text NOT NULL +); + +-- +goose Down + +DROP TABLE import_completion; diff --git a/internal/infrastructure/postgres/postgresdb/import.sql.go b/internal/infrastructure/postgres/postgresdb/import.sql.go new file mode 100644 index 000000000..c36c97386 --- /dev/null +++ b/internal/infrastructure/postgres/postgresdb/import.sql.go @@ -0,0 +1,37 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: import.sql + +package postgresdb + +import ( + "context" +) + +const importCompleted = `-- name: ImportCompleted :one +SELECT EXISTS (SELECT 1 FROM import_completion) +` + +func (q *Queries) ImportCompleted(ctx context.Context) (bool, error) { + row := q.db.QueryRow(ctx, importCompleted) + var exists bool + err := row.Scan(&exists) + return exists, err +} + +const insertImportCompletion = `-- name: InsertImportCompletion :exec + +INSERT INTO import_completion (id, sources, report) VALUES (1, $1, $2) +` + +type InsertImportCompletionParams struct { + Sources string + Report string +} + +// The one-time legacy import's completion record. +func (q *Queries) InsertImportCompletion(ctx context.Context, arg InsertImportCompletionParams) error { + _, err := q.db.Exec(ctx, insertImportCompletion, arg.Sources, arg.Report) + return err +} diff --git a/internal/infrastructure/postgres/postgresdb/models.go b/internal/infrastructure/postgres/postgresdb/models.go index 021a99135..abb6972af 100644 --- a/internal/infrastructure/postgres/postgresdb/models.go +++ b/internal/infrastructure/postgres/postgresdb/models.go @@ -119,6 +119,13 @@ type IdentitySubject struct { BoundAt time.Time } +type ImportCompletion struct { + ID int64 + CompletedAt time.Time + Sources string + Report string +} + type Mapping struct { ID string Name string diff --git a/internal/infrastructure/postgres/queries/import.sql b/internal/infrastructure/postgres/queries/import.sql new file mode 100644 index 000000000..0a741eaf2 --- /dev/null +++ b/internal/infrastructure/postgres/queries/import.sql @@ -0,0 +1,7 @@ +-- The one-time legacy import's completion record. + +-- name: InsertImportCompletion :exec +INSERT INTO import_completion (id, sources, report) VALUES (1, $1, $2); + +-- name: ImportCompleted :one +SELECT EXISTS (SELECT 1 FROM import_completion);