diff --git a/internal/infrastructure/legacyread/legacyread.go b/internal/infrastructure/legacyread/legacyread.go new file mode 100644 index 000000000..07f9206cc --- /dev/null +++ b/internal/infrastructure/legacyread/legacyread.go @@ -0,0 +1,139 @@ +// Package legacyread reads an existing install's SQLite stores (the State +// Store's archie.db, the PocketBase event store and the Gateway conversation +// store) for the one-time import into Postgres, and verifies what was read. +// +// Every production opener of those files writes on open (schema migrations, +// PocketBase bootstrap, DDL and backfills), so this package never imports +// them: it opens each file read-only and issues only SELECT. It knows nothing +// about Postgres; values come back as the SQLite driver's native types +// (int64, float64, string, []byte, nil) under their legacy column names. +// +// The package exists only for the import and is deleted with it. +package legacyread + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "strings" + + _ "modernc.org/sqlite" +) + +// Source is one legacy SQLite file opened read-only. The process that owns +// the file must be stopped: a read-only open cannot checkpoint a WAL. +type Source struct { + db *sql.DB +} + +// Open opens path read-only. It refuses a path that does not exist rather +// than letting SQLite create an empty database there. +func Open(ctx context.Context, path string) (*Source, error) { + if _, err := os.Stat(path); err != nil { + return nil, fmt.Errorf("legacyread: %w", err) + } + db, err := sql.Open("sqlite", "file:"+path+"?mode=ro&_pragma=busy_timeout(5000)") + if err != nil { + return nil, fmt.Errorf("legacyread: open %s: %w", path, err) + } + if err := db.PingContext(ctx); err != nil { + return nil, errors.Join(fmt.Errorf("legacyread: open %s: %w", path, err), db.Close()) + } + return &Source{db: db}, nil +} + +// Close closes the source. +func (s *Source) Close() error { return s.db.Close() } + +// Table names one legacy table: its columns as the production code leaves +// them (columns added by ALTER TABLE included), the columns that identify a +// row, and the order it is read in. +type Table struct { + Name string + Columns []string + Key []string + OrderBy string +} + +// Rows is one table's contents in read order. +type Rows struct { + Table Table + Rows [][]any +} + +// Value returns column col of row i, or nil for an unknown column. +func (r Rows) Value(i int, col string) any { + for j, c := range r.Table.Columns { + if c == col { + return r.Rows[i][j] + } + } + return nil +} + +// Keys returns each row's key, its key columns joined with "/", in read order. +func (r Rows) Keys() []string { + keys := make([]string, len(r.Rows)) + for i := range r.Rows { + parts := make([]string, len(r.Table.Key)) + for j, col := range r.Table.Key { + parts[j] = fmt.Sprint(r.Value(i, col)) + } + keys[i] = strings.Join(parts, "/") + } + return keys +} + +// Read returns every row of t. +func (s *Source) Read(ctx context.Context, t Table) (Rows, error) { + query := "SELECT " + strings.Join(t.Columns, ", ") + " FROM " + t.Name + " ORDER BY " + t.OrderBy + rows, err := s.db.QueryContext(ctx, query) + if err != nil { + return Rows{}, fmt.Errorf("legacyread: read %s: %w", t.Name, err) + } + defer func() { _ = rows.Close() }() + out := Rows{Table: t} + for rows.Next() { + row := make([]any, len(t.Columns)) + ptrs := make([]any, len(row)) + for i := range row { + ptrs[i] = &row[i] + } + if err := rows.Scan(ptrs...); err != nil { + return Rows{}, fmt.Errorf("legacyread: read %s: %w", t.Name, err) + } + out.Rows = append(out.Rows, row) + } + if err := rows.Err(); err != nil { + return Rows{}, fmt.Errorf("legacyread: read %s: %w", t.Name, err) + } + return out, nil +} + +// IndexedMessageIDs returns the message ids the Gateway's FTS5 search index +// holds. messages_fts is an external-content index derived from messages, so +// it is rebuilt on the target rather than copied; what the import must account +// for is whether it covered every message, which comparing this set against +// the messages ids answers. +func (s *Source) IndexedMessageIDs(ctx context.Context) ([]string, error) { + return s.strings(ctx, "SELECT CAST(id AS TEXT) FROM messages_fts_docsize ORDER BY id") +} + +func (s *Source) strings(ctx context.Context, query string) ([]string, error) { + rows, err := s.db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("legacyread: %w", err) + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + return nil, fmt.Errorf("legacyread: %w", err) + } + out = append(out, v) + } + return out, rows.Err() +} diff --git a/internal/infrastructure/legacyread/legacyread_test.go b/internal/infrastructure/legacyread/legacyread_test.go new file mode 100644 index 000000000..a1d1ff867 --- /dev/null +++ b/internal/infrastructure/legacyread/legacyread_test.go @@ -0,0 +1,449 @@ +package legacyread + +import ( + "crypto/sha256" + "database/sql" + "os" + "path/filepath" + "slices" + "testing" +) + +// The fixtures below apply the legacy schemas as their production openers +// leave them, columns added by ALTER TABLE included, without importing those +// openers. +const stateStoreDDL = ` +CREATE TABLE tasks (id INTEGER PRIMARY KEY AUTOINCREMENT, owner TEXT NOT NULL, repo TEXT NOT NULL, + issue_number INTEGER NOT NULL, title TEXT DEFAULT '', body TEXT DEFAULT '', labels TEXT DEFAULT '', + status TEXT DEFAULT 'queued', workflow TEXT DEFAULT '', stage TEXT DEFAULT '', branch TEXT DEFAULT '', + plan TEXT DEFAULT '', notes TEXT DEFAULT '', pr_number INTEGER DEFAULT 0, tokens_used INTEGER DEFAULT 0, + iterations INTEGER DEFAULT 0, attempt INTEGER DEFAULT 0, park_reason TEXT DEFAULT '', + watch_comment_id INTEGER DEFAULT 0, park_class TEXT DEFAULT 'needs_human', remediation_rounds INTEGER DEFAULT 0, + retry_count INTEGER DEFAULT 0, source TEXT DEFAULT 'forge', identity TEXT DEFAULT '', binding_id TEXT DEFAULT '', + binding_version INTEGER DEFAULT 0, review_payload TEXT DEFAULT '', workflow_definition_version INTEGER DEFAULT 0, + workflow_definition_digest TEXT DEFAULT '', workflow_definition_yaml TEXT DEFAULT '', + created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now'))); +ALTER TABLE tasks ADD COLUMN review_cursor INTEGER NOT NULL DEFAULT 0; +CREATE TABLE transitions (id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL, + at TEXT DEFAULT (datetime('now')), from_status TEXT NOT NULL, to_status TEXT NOT NULL, detail TEXT DEFAULT ''); +CREATE TABLE events (id INTEGER PRIMARY KEY AUTOINCREMENT, at TEXT NOT NULL, kind TEXT NOT NULL, + task_id INTEGER DEFAULT 0, repo TEXT DEFAULT '', issue INTEGER DEFAULT 0, workflow TEXT DEFAULT '', + stage TEXT DEFAULT '', attempt INTEGER DEFAULT 0, actor_id TEXT DEFAULT '', actor_kind TEXT DEFAULT '', + principal_id TEXT DEFAULT '', detail TEXT DEFAULT '', data TEXT DEFAULT '{}'); +CREATE TABLE resources (kind TEXT PRIMARY KEY, value BLOB NOT NULL, version INTEGER NOT NULL, updated_at TEXT NOT NULL); +CREATE TABLE resource_history (id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, value BLOB NOT NULL, + version INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL, request_id TEXT NOT NULL, + expected_version INTEGER NOT NULL, current_version INTEGER NOT NULL, at TEXT NOT NULL); +CREATE TABLE channel_status (id TEXT PRIMARY KEY, name TEXT DEFAULT '', state TEXT DEFAULT '', detail TEXT DEFAULT '', + configured INTEGER NOT NULL DEFAULT 0, reload_supported INTEGER NOT NULL DEFAULT 0, observed_at TEXT NOT NULL); +CREATE TABLE apply_status (process TEXT NOT NULL, kind TEXT NOT NULL, applied_version INTEGER DEFAULT 0, + error TEXT DEFAULT '', reported_at TEXT NOT NULL, PRIMARY KEY (process, kind)); +CREATE TABLE config_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), schema TEXT DEFAULT '', document TEXT DEFAULT '', + published_at TEXT NOT NULL); +CREATE TABLE identities (id TEXT PRIMARY KEY, kind TEXT NOT NULL, display_name TEXT NOT NULL, lifecycle TEXT NOT NULL, + version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL); +CREATE TABLE identity_aliases (alias TEXT PRIMARY KEY COLLATE NOCASE, identity_id TEXT NOT NULL REFERENCES identities(id)); +CREATE TABLE identity_events (id INTEGER PRIMARY KEY AUTOINCREMENT, identity_id TEXT NOT NULL, event_type TEXT NOT NULL, + from_lifecycle TEXT NOT NULL, to_lifecycle TEXT NOT NULL, display_name TEXT NOT NULL, actor_id TEXT NOT NULL, + source TEXT NOT NULL, request_id TEXT NOT NULL UNIQUE, at TEXT NOT NULL); +CREATE TABLE identity_subjects (issuer TEXT NOT NULL, subject TEXT NOT NULL, + identity_id TEXT NOT NULL REFERENCES identities(id), bound_at TEXT NOT NULL, PRIMARY KEY (issuer, subject)); + +INSERT INTO tasks (id, owner, repo, issue_number, review_cursor) VALUES (1, 'o', 'r', 7, 3), (4, 'o', 'r', 1000000000000000, 0); +INSERT INTO transitions (task_id, at, from_status, to_status) VALUES (1, '2026-01-01 00:00:00', 'queued', 'running'); +INSERT INTO events (id, at, kind) VALUES + (1, '2026-01-01T00:00:00.000000100Z', 'a'), + (2, '2026-01-01T00:00:01.000000000Z', 'b'); +INSERT INTO resources VALUES ('config', X'00ff10', 2, '2026-01-01T00:00:00Z'); +INSERT INTO resource_history (kind, value, version, actor, source, request_id, expected_version, current_version, at) + VALUES ('config', X'00ff10', 2, 'sam', 'ui', 'req-1', 1, 1, '2026-01-01T00:00:00Z'); +INSERT INTO channel_status VALUES ('tg', 'Telegram', 'up', '', 1, 0, '2026-01-01T00:00:00Z'); +INSERT INTO apply_status VALUES ('daemon', 'config', 2, '', '2026-01-01T00:00:00Z'); +INSERT INTO config_snapshot VALUES (1, 's', '{}', '2026-01-01T00:00:00Z'); +INSERT INTO identities VALUES ('id-1', 'agent', 'Archie', 'active', 1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z'); +INSERT INTO identity_aliases VALUES ('archie', 'id-1'); +INSERT INTO identity_events (identity_id, event_type, from_lifecycle, to_lifecycle, display_name, actor_id, source, request_id, at) + VALUES ('id-1', 'created', '', 'active', 'Archie', 'sam', 'boot', 'rq-1', '2026-01-01T00:00:00Z'); +INSERT INTO identity_subjects VALUES ('https://issuer', 'sub-1', 'id-1', '2026-01-01T00:00:00Z'); +` + +const edaDDL = ` +CREATE TABLE captures (id TEXT PRIMARY KEY NOT NULL, authenticated BOOLEAN DEFAULT FALSE NOT NULL, + body TEXT DEFAULT '' NOT NULL, content_type TEXT DEFAULT '' NOT NULL, headers JSON DEFAULT NULL, + received_at TEXT DEFAULT '' NOT NULL, remote_addr TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL); +CREATE TABLE mappings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, fields JSON DEFAULT NULL, + name TEXT DEFAULT '' NOT NULL, source_hint TEXT DEFAULT '' NOT NULL, updated_at TEXT DEFAULT '' NOT NULL); +CREATE TABLE bindings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, mapping TEXT DEFAULT '' NOT NULL, + name TEXT DEFAULT '' NOT NULL, owner TEXT DEFAULT '' NOT NULL, repo TEXT DEFAULT '' NOT NULL, + secret TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL, status TEXT DEFAULT '' NOT NULL, + updated_at TEXT DEFAULT '' NOT NULL, version NUMERIC DEFAULT 0 NOT NULL, workflow TEXT DEFAULT '' NOT NULL); +CREATE TABLE binding_dispatches (id TEXT PRIMARY KEY NOT NULL, binding TEXT DEFAULT '' NOT NULL, + binding_version NUMERIC DEFAULT 0 NOT NULL, capture TEXT DEFAULT '' NOT NULL, dispatched_at TEXT DEFAULT '' NOT NULL, + task_id NUMERIC DEFAULT 0 NOT NULL); +CREATE UNIQUE INDEX idx_binding_dispatch_once ON binding_dispatches (binding, capture); +CREATE TABLE playbook_dispatches (id TEXT PRIMARY KEY NOT NULL, action_id TEXT DEFAULT '' NOT NULL, + dispatched_at TEXT DEFAULT '' NOT NULL, event_id TEXT DEFAULT '' NOT NULL, playbook_id TEXT DEFAULT '' NOT NULL, + playbook_version TEXT DEFAULT '' NOT NULL); +CREATE TABLE tool_calls (id TEXT PRIMARY KEY NOT NULL, args JSON DEFAULT NULL, attempt NUMERIC DEFAULT 0 NOT NULL, + called_at TEXT DEFAULT '' NOT NULL, duration_ms NUMERIC DEFAULT 0 NOT NULL, error TEXT DEFAULT '' NOT NULL, + result JSON DEFAULT NULL, task_id NUMERIC DEFAULT 0 NOT NULL, tool TEXT DEFAULT '' NOT NULL); + +INSERT INTO captures VALUES ('rcap1', 1, '{}', 'application/json', '{"x":["1"]}', '2026-01-01T00:00:00.000000000Z', '127.0.0.1', 'gh'); +INSERT INTO mappings VALUES ('rmap1', '2026-01-01', '[]', 'm', 'gh', '2026-01-01'); +INSERT INTO bindings VALUES ('rbind1', '2026-01-01', 'rmap1', 'b', 'o', 'r', 'plaintext-secret', 'gh', 'armed', '2026-01-01', 1, 'implement'); +INSERT INTO binding_dispatches VALUES ('rdisp1', 'rbind1', 1, 'rcap1', '2026-01-01', 4); +INSERT INTO playbook_dispatches VALUES ('rpb1', 'act', '2026-01-01', 'ev', 'pb', 'v1'); +INSERT INTO tool_calls VALUES ('rtool1', '{}', 1, '2026-01-01', 12, '', '{}', 4, 'bash'); +` + +const gatewayDDL = ` +CREATE TABLE sessions (session_id TEXT PRIMARY KEY, platform TEXT NOT NULL DEFAULT '', bot_user TEXT NOT NULL DEFAULT '', + channel_id TEXT NOT NULL DEFAULT '', thread_id TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', + parent_session_id TEXT NOT NULL DEFAULT '', branch_name TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL DEFAULT 0, last_active_at INTEGER NOT NULL DEFAULT 0); +CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT NOT NULL, session_id TEXT NOT NULL, + source_id TEXT NOT NULL DEFAULT '', sender TEXT NOT NULL DEFAULT '', text TEXT NOT NULL DEFAULT '', ts INTEGER NOT NULL, + UNIQUE(session_id, message_id)); +ALTER TABLE messages ADD COLUMN role TEXT NOT NULL DEFAULT ''; +ALTER TABLE messages ADD COLUMN sender_id TEXT NOT NULL DEFAULT ''; +CREATE VIRTUAL TABLE messages_fts USING fts5(sender, text, content='messages', content_rowid='id'); +CREATE TRIGGER messages_ai AFTER INSERT ON messages BEGIN + INSERT INTO messages_fts(rowid, sender, text) VALUES (new.id, new.sender, new.text); +END; +CREATE TABLE turns (turn_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, source_id TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL, attempt INTEGER NOT NULL DEFAULT 0, owner_id TEXT NOT NULL DEFAULT '', + input_message_id TEXT NOT NULL DEFAULT '', assistant_message_id TEXT NOT NULL DEFAULT '', + partial_text TEXT NOT NULL DEFAULT '', response_text TEXT NOT NULL DEFAULT '', tool_calls TEXT NOT NULL DEFAULT '[]', + error TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL); + +INSERT INTO sessions VALUES ('s1', 'telegram', 'bot', 'c1', '', 'Chat', '', '', 1000, 2000); +INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m1', 's1', 'sam', 'hello there', 1000, 'user'); +INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m2', 's1', 'archie', 'hi', 1001, 'assistant'); +INSERT INTO turns VALUES ('t1', 's1', '', 'done', 1, '', 'm1', 'm2', '', 'hi', '[]', '', 1000, 1001); +` + +// fixture writes a legacy database built from ddl and returns its path. It +// is checkpointed into a single file so a hash covers all of its state. +func fixture(t *testing.T, name, ddl string) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer func() { _ = db.Close() }() + if _, err := db.ExecContext(t.Context(), ddl); err != nil { + t.Fatalf("apply fixture DDL: %v", err) + } + return path +} + +func exec(t *testing.T, path, stmt string) { + t.Helper() + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + t.Fatalf("open: %v", err) + } + defer func() { _ = db.Close() }() + if _, err := db.ExecContext(t.Context(), stmt); err != nil { + t.Fatalf("exec %q: %v", stmt, err) + } +} + +func hashDir(t *testing.T, dir string) map[string][32]byte { + t.Helper() + sums := map[string][32]byte{} + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read dir: %v", err) + } + for _, e := range entries { + b, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + t.Fatalf("read %s: %v", e.Name(), err) + } + sums[e.Name()] = sha256.Sum256(b) + } + return sums +} + +func openSource(t *testing.T, path string) *Source { + t.Helper() + src, err := Open(t.Context(), path) + if err != nil { + t.Fatalf("Open(%s): %v", path, err) + } + t.Cleanup(func() { _ = src.Close() }) + return src +} + +// TestReadersDoNotMutateSources reads every table of all three stores and +// asserts every file in each store's directory is byte-for-byte unchanged, +// including the FTS5 index read. +func TestReadersDoNotMutateSources(t *testing.T) { + stores := []struct { + name string + ddl string + tables []Table + rows map[string]int + }{ + {"archie.db", stateStoreDDL, StateStoreTables, map[string]int{"tasks": 2, "events": 2, "identity_subjects": 1}}, + {"archie.db-eda.sqlite", edaDDL, EDATables, map[string]int{"captures": 1, "bindings": 1, "tool_calls": 1}}, + {"archie.db-conversations.sqlite", gatewayDDL, GatewayTables, map[string]int{"messages": 2, "turns": 1}}, + } + for _, s := range stores { + t.Run(s.name, func(t *testing.T) { + path := fixture(t, s.name, s.ddl) + before := hashDir(t, filepath.Dir(path)) + src := openSource(t, path) + for _, table := range s.tables { + rows, err := src.Read(t.Context(), table) + if err != nil { + t.Fatalf("Read(%s): %v", table.Name, err) + } + if want, ok := s.rows[table.Name]; ok && len(rows.Rows) != want { + t.Errorf("Read(%s) = %d rows, want %d", table.Name, len(rows.Rows), want) + } + } + if s.name == "archie.db-conversations.sqlite" { + if _, err := src.IndexedMessageIDs(t.Context()); err != nil { + t.Fatalf("IndexedMessageIDs: %v", err) + } + } + if err := src.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if after := hashDir(t, filepath.Dir(path)); !mapsEqual(before, after) { + t.Fatalf("reading changed the source directory:\nbefore %x\nafter %x", before, after) + } + }) + } +} + +func mapsEqual(a, b map[string][32]byte) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +// TestOpenRefusesWrites pins the read-only open: a write through the +// source's connection is an error, not a silent mutation. +func TestOpenRefusesWrites(t *testing.T) { + src := openSource(t, fixture(t, "archie.db", stateStoreDDL)) + if _, err := src.db.ExecContext(t.Context(), "DELETE FROM tasks"); err == nil { + t.Fatal("DELETE through the read-only source succeeded") + } +} + +func TestReadCarriesAlteredColumnsAndBlobs(t *testing.T) { + src := openSource(t, fixture(t, "archie.db", stateStoreDDL)) + tasks, err := src.Read(t.Context(), tableNamed(t, StateStoreTables, "tasks")) + if err != nil { + t.Fatalf("Read(tasks): %v", err) + } + if got := tasks.Value(0, "review_cursor"); got != int64(3) { + t.Errorf("tasks[0].review_cursor = %#v, want 3 (an ALTER TABLE column)", got) + } + if got := tasks.Value(1, "issue_number"); got != int64(1_000_000_000_000_000) { + t.Errorf("tasks[1].issue_number = %#v, want the chat seed", got) + } + res, err := src.Read(t.Context(), tableNamed(t, StateStoreTables, "resources")) + if err != nil { + t.Fatalf("Read(resources): %v", err) + } + if got, ok := res.Value(0, "value").([]byte); !ok || string(got) != "\x00\xff\x10" { + t.Errorf("resources[0].value = %#v, want the raw bytes 00 ff 10", res.Value(0, "value")) + } +} + +func tableNamed(t *testing.T, tables []Table, name string) Table { + t.Helper() + i := slices.IndexFunc(tables, func(tb Table) bool { return tb.Name == name }) + if i < 0 { + t.Fatalf("no table %q", name) + } + return tables[i] +} + +func TestCompareKeys(t *testing.T) { + tests := []struct { + name string + source []string + target []string + missing, extra []string + }{ + {name: "equal with the same gap", source: []string{"1", "2", "5"}, target: []string{"5", "1", "2"}}, + {name: "swallowed row", source: []string{"1", "2", "3"}, target: []string{"1", "3"}, missing: []string{"2"}}, + {name: "same count, different ids", source: []string{"1", "2"}, target: []string{"1", "9"}, missing: []string{"2"}, extra: []string{"9"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := CompareKeys(tt.source, tt.target) + if !slices.Equal(got.Missing, tt.missing) || !slices.Equal(got.Extra, tt.extra) { + t.Errorf("CompareKeys = %+v, want missing %v extra %v", got, tt.missing, tt.extra) + } + }) + } +} + +func TestCompareOrder(t *testing.T) { + if got := CompareOrder([]int64{1, 2, 3}, []int64{1, 2, 3}); got != nil { + t.Errorf("CompareOrder(equal) = %+v, want nil", got) + } + got := CompareOrder([]int64{1, 2, 3}, []int64{1, 3, 2}) + if got == nil || got.Position != 1 || got.Source != 2 || got.Target != 3 { + t.Errorf("CompareOrder(swapped) = %+v, want the first divergence at position 1 (2 vs 3)", got) + } + if got := CompareOrder([]int64{1, 2}, []int64{1}); got == nil || got.Position != 1 { + t.Errorf("CompareOrder(short target) = %+v, want a divergence at position 1", got) + } +} + +// TestMicrosecondCollisions constructs D3.8's condition: two events whose +// instants differ only below a microsecond, with the later instant carrying +// the lower id, so Postgres's (at, id) order would reverse them. +func TestMicrosecondCollisions(t *testing.T) { + path := fixture(t, "archie.db", stateStoreDDL) + exec(t, path, `INSERT INTO events (id, at, kind) VALUES + (10, '2026-02-01T00:00:00.000000900Z', 'later instant, lower id'), + (11, '2026-02-01T00:00:00.000000100Z', 'earlier instant, higher id'), + (12, '2026-02-01T00:00:05.000000100Z', 'same microsecond, ids in order'), + (13, '2026-02-01T00:00:05.000000900Z', 'same microsecond, ids in order')`) + src := openSource(t, path) + got, err := src.MicrosecondCollisions(t.Context()) + if err != nil { + t.Fatalf("MicrosecondCollisions: %v", err) + } + if len(got) != 1 || got[0].EarlierID != 11 || got[0].LaterID != 10 { + t.Fatalf("MicrosecondCollisions = %+v, want exactly the pair (11 before 10)", got) + } +} + +func TestOrphans(t *testing.T) { + path := fixture(t, "archie.db", stateStoreDDL) + src := openSource(t, path) + for _, fk := range IdentityForeignKeys { + got, err := src.Orphans(t.Context(), fk) + if err != nil || len(got) != 0 { + t.Fatalf("Orphans(%s) on a clean fixture = %v, %v; want none", fk.Table, got, err) + } + } + _ = src.Close() + exec(t, path, `INSERT INTO identity_aliases VALUES ('ghost', 'id-missing')`) + src = openSource(t, path) + got, err := src.Orphans(t.Context(), IdentityForeignKeys[0]) + if err != nil { + t.Fatalf("Orphans: %v", err) + } + if len(got) != 1 || got[0].Key != "ghost" || got[0].Parent != "id-missing" { + t.Fatalf("Orphans(identity_aliases) = %+v, want the ghost alias", got) + } +} + +// TestDuplicateAliases pins the case-folding hazard: SQLite's NOCASE folds +// ASCII only, so two aliases differing in non-ASCII case coexist there and +// collide under Postgres's Unicode lower(). +func TestDuplicateAliases(t *testing.T) { + path := fixture(t, "archie.db", stateStoreDDL) + exec(t, path, `INSERT INTO identity_aliases VALUES ('ÉMILE', 'id-1'), ('émile', 'id-1')`) + src := openSource(t, path) + got, err := src.DuplicateAliases(t.Context()) + if err != nil { + t.Fatalf("DuplicateAliases: %v", err) + } + if len(got) != 1 || got[0].Key != "émile" || len(got[0].Values) != 2 { + t.Fatalf("DuplicateAliases = %+v, want one group of two for émile", got) + } +} + +func TestDuplicateArmedBindings(t *testing.T) { + path := fixture(t, "archie.db-eda.sqlite", edaDDL) + src := openSource(t, path) + if got, err := src.DuplicateBindings(t.Context()); err != nil || len(got) != 0 { + t.Fatalf("DuplicateBindings on a clean fixture = %v, %v; want none", got, err) + } + _ = src.Close() + exec(t, path, `INSERT INTO bindings VALUES ('rbind2', '', 'rmap1', 'b2', 'o', 'r', '', 'gh', 'pending_approval', '', 1, 'implement')`) + src = openSource(t, path) + got, err := src.DuplicateBindings(t.Context()) + if err != nil { + t.Fatalf("DuplicateBindings: %v", err) + } + if len(got) != 1 || got[0].Key != "gh" || !slices.Equal(got[0].Values, []string{"rbind1", "rbind2"}) { + t.Fatalf("DuplicateBindings = %+v, want source gh held by rbind1 and rbind2", got) + } +} + +func TestSequences(t *testing.T) { + src := openSource(t, fixture(t, "archie.db", stateStoreDDL)) + got, err := src.Sequences(t.Context()) + if err != nil { + t.Fatalf("Sequences: %v", err) + } + want := map[string]Sequence{ + "tasks": {Table: "tasks", MaxID: 4, Next: 5}, + "transitions": {Table: "transitions", MaxID: 1, Next: 2}, + "events": {Table: "events", MaxID: 2, Next: 3}, + "resource_history": {Table: "resource_history", MaxID: 1, Next: 2}, + "identity_events": {Table: "identity_events", MaxID: 1, Next: 2}, + } + if len(got) != len(want) { + t.Fatalf("Sequences = %+v, want %d tables", got, len(want)) + } + for _, s := range got { + if want[s.Table] != s { + t.Errorf("Sequences[%s] = %+v, want %+v", s.Table, s, want[s.Table]) + } + } +} + +// TestSequenceNextSkipsDeletedTop pins that Next comes from AUTOINCREMENT's +// high-water mark, not max(id): a deleted top row still reserves its id. +func TestSequenceNextSkipsDeletedTop(t *testing.T) { + path := fixture(t, "archie.db", stateStoreDDL) + exec(t, path, `DELETE FROM tasks WHERE id = 4`) + src := openSource(t, path) + got, err := src.Sequences(t.Context()) + if err != nil { + t.Fatalf("Sequences: %v", err) + } + i := slices.IndexFunc(got, func(s Sequence) bool { return s.Table == "tasks" }) + if got[i].MaxID != 1 || got[i].Next != 5 { + t.Fatalf("Sequences[tasks] = %+v, want max 1, next 5", got[i]) + } +} + +func TestCompareBlobs(t *testing.T) { + src := map[string][]byte{"config": {0, 0xff}, "routing": {1}} + if got := CompareBlobs(src, map[string][]byte{"config": {0, 0xff}, "routing": {1}}); len(got) != 0 { + t.Errorf("CompareBlobs(equal) = %v, want none", got) + } + got := CompareBlobs(src, map[string][]byte{"config": {0, 0xfe}}) + if !slices.Equal(got, []string{"config", "routing"}) { + t.Errorf("CompareBlobs = %v, want config (changed) and routing (missing)", got) + } +} + +func TestIndexedMessageIDsFindsAStaleIndex(t *testing.T) { + path := fixture(t, "archie.db-conversations.sqlite", gatewayDDL) + exec(t, path, `DROP TRIGGER messages_ai; INSERT INTO messages (message_id, session_id, text, ts) VALUES ('m3', 's1', 'unindexed', 1002)`) + src := openSource(t, path) + indexed, err := src.IndexedMessageIDs(t.Context()) + if err != nil { + t.Fatalf("IndexedMessageIDs: %v", err) + } + messages, err := src.Read(t.Context(), tableNamed(t, GatewayTables, "messages")) + if err != nil { + t.Fatalf("Read(messages): %v", err) + } + got := CompareKeys(messages.Keys(), indexed) + if !slices.Equal(got.Missing, []string{"3"}) { + t.Fatalf("CompareKeys(messages, index) = %+v, want message 3 missing from the index", got) + } +} diff --git a/internal/infrastructure/legacyread/tables.go b/internal/infrastructure/legacyread/tables.go new file mode 100644 index 000000000..eb7fb6204 --- /dev/null +++ b/internal/infrastructure/legacyread/tables.go @@ -0,0 +1,73 @@ +package legacyread + +func idTable(name string, columns ...string) Table { + return Table{Name: name, Columns: columns, Key: []string{columns[0]}, OrderBy: columns[0]} +} + +// StateStoreTables are the twelve tables of the State Store's archie.db. +var StateStoreTables = []Table{ + idTable("tasks", "id", "owner", "repo", "issue_number", "title", "body", "labels", "status", + "workflow", "stage", "branch", "plan", "notes", "pr_number", "tokens_used", "iterations", + "attempt", "park_reason", "watch_comment_id", "park_class", "remediation_rounds", "retry_count", + "source", "identity", "binding_id", "binding_version", "review_payload", + "workflow_definition_version", "workflow_definition_digest", "workflow_definition_yaml", + "created_at", "updated_at", "review_cursor"), + idTable("transitions", "id", "task_id", "at", "from_status", "to_status", "detail"), + { + Name: "events", + Columns: []string{ + "id", "at", "kind", "task_id", "repo", "issue", "workflow", "stage", "attempt", + "actor_id", "actor_kind", "principal_id", "detail", "data", + }, + Key: []string{"id"}, + OrderBy: "at, id", + }, + idTable("resources", "kind", "value", "version", "updated_at"), + idTable("resource_history", "id", "kind", "value", "version", "actor", "source", "request_id", + "expected_version", "current_version", "at"), + idTable("channel_status", "id", "name", "state", "detail", "configured", "reload_supported", "observed_at"), + { + Name: "apply_status", + Columns: []string{"process", "kind", "applied_version", "error", "reported_at"}, + Key: []string{"process", "kind"}, + OrderBy: "process, kind", + }, + idTable("config_snapshot", "id", "schema", "document", "published_at"), + idTable("identities", "id", "kind", "display_name", "lifecycle", "version", "created_at", "updated_at"), + idTable("identity_aliases", "alias", "identity_id"), + idTable("identity_events", "id", "identity_id", "event_type", "from_lifecycle", "to_lifecycle", + "display_name", "actor_id", "source", "request_id", "at"), + { + Name: "identity_subjects", + Columns: []string{"issuer", "subject", "identity_id", "bound_at"}, + Key: []string{"issuer", "subject"}, + OrderBy: "issuer, subject", + }, +} + +// EDATables are the six app collections of the PocketBase event store. +// PocketBase's own tables (_collections, _params, _superusers, ...) are not +// archie data and are not read. +var EDATables = []Table{ + idTable("captures", "id", "authenticated", "body", "content_type", "headers", "received_at", + "remote_addr", "source"), + idTable("mappings", "id", "created_at", "fields", "name", "source_hint", "updated_at"), + idTable("bindings", "id", "created_at", "mapping", "name", "owner", "repo", "secret", "source", + "status", "updated_at", "version", "workflow"), + idTable("binding_dispatches", "id", "binding", "binding_version", "capture", "dispatched_at", "task_id"), + idTable("playbook_dispatches", "id", "action_id", "dispatched_at", "event_id", "playbook_id", + "playbook_version"), + idTable("tool_calls", "id", "args", "attempt", "called_at", "duration_ms", "error", "result", + "task_id", "tool"), +} + +// GatewayTables are the Gateway conversation store's three data tables. Its +// FTS5 index is read through IndexedMessageIDs. +var GatewayTables = []Table{ + idTable("sessions", "session_id", "platform", "bot_user", "channel_id", "thread_id", "title", + "parent_session_id", "branch_name", "created_at", "last_active_at"), + idTable("messages", "id", "message_id", "session_id", "source_id", "sender", "text", "ts", "role", "sender_id"), + idTable("turns", "turn_id", "session_id", "source_id", "status", "attempt", "owner_id", + "input_message_id", "assistant_message_id", "partial_text", "response_text", "tool_calls", + "error", "created_at", "updated_at"), +} diff --git a/internal/infrastructure/legacyread/verify.go b/internal/infrastructure/legacyread/verify.go new file mode 100644 index 000000000..d3ecc1435 --- /dev/null +++ b/internal/infrastructure/legacyread/verify.go @@ -0,0 +1,275 @@ +package legacyread + +import ( + "bytes" + "context" + "fmt" + "maps" + "slices" + "strings" + "time" +) + +// KeyDiff is the difference between two key sets. A count comparison alone +// passes on a swallowed row when ids have gaps; comparing the sets does not. +type KeyDiff struct { + Missing []string // in the source, not the target + Extra []string // in the target, not the source +} + +// CompareKeys compares the source's key set with the target's. +func CompareKeys(source, target []string) KeyDiff { + var d KeyDiff + src := setOf(source) + tgt := setOf(target) + for k := range src { + if !tgt[k] { + d.Missing = append(d.Missing, k) + } + } + for k := range tgt { + if !src[k] { + d.Extra = append(d.Extra, k) + } + } + slices.Sort(d.Missing) + slices.Sort(d.Extra) + return d +} + +func setOf(keys []string) map[string]bool { + set := make(map[string]bool, len(keys)) + for _, k := range keys { + set[k] = true + } + return set +} + +// Divergence is the first position at which two ordered id sequences differ. +// A missing element on one side reads as 0. +type Divergence struct { + Position int + Source, Target int64 +} + +// CompareOrder compares two id sequences read in cursor order, returning nil +// when they are identical. +func CompareOrder(source, target []int64) *Divergence { + for i := range max(len(source), len(target)) { + var s, t int64 + if i < len(source) { + s = source[i] + } + if i < len(target) { + t = target[i] + } + if s != t || i >= len(source) || i >= len(target) { + return &Divergence{Position: i, Source: s, Target: t} + } + } + return nil +} + +// Collision is two events whose instants differ only below a microsecond +// while their ids run the other way. Postgres stores microseconds, so its +// (at, id) order would put LaterID before EarlierID. +type Collision struct { + EarlierID, LaterID int64 + At time.Time // the shared microsecond +} + +// eventAtLayouts are the layouts events.at has been written in, newest first. +var eventAtLayouts = []string{"2006-01-02T15:04:05.000000000Z", time.RFC3339Nano, time.RFC3339} + +func parseEventAt(s string) (time.Time, error) { + for _, layout := range eventAtLayouts { + if t, err := time.Parse(layout, s); err == nil { + return t.UTC(), nil + } + } + return time.Time{}, fmt.Errorf("unrecognised timestamp %q", s) +} + +// MicrosecondCollisions returns every event pair that truncating events.at +// to microseconds would reorder. +func (s *Source) MicrosecondCollisions(ctx context.Context) ([]Collision, error) { + rows, err := s.db.QueryContext(ctx, "SELECT id, at FROM events ORDER BY id") + if err != nil { + return nil, fmt.Errorf("legacyread: events: %w", err) + } + defer func() { _ = rows.Close() }() + type event struct { + id int64 + at time.Time + } + byMicro := map[time.Time][]event{} + for rows.Next() { + var ( + id int64 + raw string + ) + if err := rows.Scan(&id, &raw); err != nil { + return nil, fmt.Errorf("legacyread: events: %w", err) + } + at, err := parseEventAt(raw) + if err != nil { + return nil, fmt.Errorf("legacyread: event %d: %w", id, err) + } + micro := at.Truncate(time.Microsecond) + byMicro[micro] = append(byMicro[micro], event{id, at}) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("legacyread: events: %w", err) + } + var out []Collision + for micro, events := range byMicro { + for i, a := range events { + for _, b := range events[i+1:] { + // Read in id order, so a.id < b.id. + if b.at.Before(a.at) { + out = append(out, Collision{EarlierID: b.id, LaterID: a.id, At: micro}) + } + } + } + } + slices.SortFunc(out, func(x, y Collision) int { return int(x.EarlierID - y.EarlierID) }) + return out, nil +} + +// ForeignKey is a reference SQLite declared but never enforced (the State +// Store never set PRAGMA foreign_keys) and Postgres will. +type ForeignKey struct { + Table, Column string + KeyExpr string // SQL naming the child row in a report + Parent, ParentC string +} + +// IdentityForeignKeys are the three identity references. +var IdentityForeignKeys = []ForeignKey{ + {Table: "identity_aliases", Column: "identity_id", KeyExpr: "alias", Parent: "identities", ParentC: "id"}, + {Table: "identity_events", Column: "identity_id", KeyExpr: "CAST(id AS TEXT)", Parent: "identities", ParentC: "id"}, + {Table: "identity_subjects", Column: "identity_id", KeyExpr: "issuer || '/' || subject", Parent: "identities", ParentC: "id"}, +} + +// Orphan is a child row whose parent does not exist. +type Orphan struct { + Key, Parent string +} + +// Orphans returns fk's child rows with no parent. +func (s *Source) Orphans(ctx context.Context, fk ForeignKey) ([]Orphan, error) { + query := fmt.Sprintf("SELECT %s, %s FROM %s WHERE %s NOT IN (SELECT %s FROM %s) ORDER BY 1", + fk.KeyExpr, fk.Column, fk.Table, fk.Column, fk.ParentC, fk.Parent) + rows, err := s.db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("legacyread: orphans in %s: %w", fk.Table, err) + } + defer func() { _ = rows.Close() }() + var out []Orphan + for rows.Next() { + var o Orphan + if err := rows.Scan(&o.Key, &o.Parent); err != nil { + return nil, fmt.Errorf("legacyread: orphans in %s: %w", fk.Table, err) + } + out = append(out, o) + } + return out, rows.Err() +} + +// Duplicate is a key the target will hold unique that more than one source +// row shares. +type Duplicate struct { + Key string + Values []string +} + +// DuplicateAliases groups identity aliases by Unicode lower case. SQLite's +// NOCASE folds ASCII only, so aliases that differ in non-ASCII case coexist in +// the source and collide on the target's lower(alias) index. +func (s *Source) DuplicateAliases(ctx context.Context) ([]Duplicate, error) { + aliases, err := s.strings(ctx, "SELECT alias FROM identity_aliases ORDER BY alias") + if err != nil { + return nil, err + } + return duplicates(aliases, aliases, strings.ToLower), nil +} + +// DuplicateBindings groups bindings by source. One binding per source is +// enforced in Go, not by the legacy schema, so a duplicate is possible and is +// refused rather than repaired (D4.2). +func (s *Source) DuplicateBindings(ctx context.Context) ([]Duplicate, error) { + rows, err := s.Read(ctx, Table{Name: "bindings", Columns: []string{"id", "source"}, OrderBy: "id"}) + if err != nil { + return nil, err + } + ids := make([]string, len(rows.Rows)) + sources := make([]string, len(rows.Rows)) + for i := range rows.Rows { + ids[i] = fmt.Sprint(rows.Value(i, "id")) + sources[i] = fmt.Sprint(rows.Value(i, "source")) + } + return duplicates(sources, ids, func(s string) string { return s }), nil +} + +// duplicates groups values by key(keys[i]) and returns the groups of two or +// more, sorted by key. +func duplicates(keys, values []string, key func(string) string) []Duplicate { + groups := map[string][]string{} + for i, k := range keys { + groups[key(k)] = append(groups[key(k)], values[i]) + } + var out []Duplicate + for _, k := range slices.Sorted(maps.Keys(groups)) { + if len(groups[k]) > 1 { + out = append(out, Duplicate{Key: k, Values: groups[k]}) + } + } + return out +} + +// Sequence is an identity column's state: the highest id present and the id +// SQLite would assign next. Next comes from AUTOINCREMENT's high-water mark, +// so a deleted top row still reserves its id. +type Sequence struct { + Table string + MaxID, Next int64 +} + +// sequenceTables are the State Store tables with an AUTOINCREMENT id. +var sequenceTables = []string{"tasks", "transitions", "events", "resource_history", "identity_events"} + +// Sequences returns the state of every State Store identity column. +func (s *Source) Sequences(ctx context.Context) ([]Sequence, error) { + out := make([]Sequence, 0, len(sequenceTables)) + for _, table := range sequenceTables { + seq := Sequence{Table: table} + var high int64 + err := s.db.QueryRowContext(ctx, fmt.Sprintf( + "SELECT COALESCE((SELECT max(id) FROM %[1]s), 0), COALESCE((SELECT seq FROM sqlite_sequence WHERE name = '%[1]s'), 0)", + table)).Scan(&seq.MaxID, &high) + if err != nil { + return nil, fmt.Errorf("legacyread: sequence %s: %w", table, err) + } + seq.Next = max(seq.MaxID, high) + 1 + out = append(out, seq) + } + return out, nil +} + +// CompareBlobs returns, sorted, every key whose bytes differ between source +// and target or that only one side holds. +func CompareBlobs(source, target map[string][]byte) []string { + var out []string + for k, v := range source { + if t, ok := target[k]; !ok || !bytes.Equal(v, t) { + out = append(out, k) + } + } + for k := range target { + if _, ok := source[k]; !ok { + out = append(out, k) + } + } + slices.Sort(out) + return out +}