feat(legacyread): read-only readers and import verification for the legacy SQLite stores

This commit is contained in:
sam committed 2026-09-24 03:28:42 +10:00
1 parent 92c116f606
commit f9dc76751c
4 files changed
+936

No files matched your search

@@ -0,0 +1,139 @@
// Package legacyread reads an existing install's SQLite stores (the State
// Store's archie.db, the PocketBase event store and the Gateway conversation
// store) for the one-time import into Postgres, and verifies what was read.
//
// Every production opener of those files writes on open (schema migrations,
// PocketBase bootstrap, DDL and backfills), so this package never imports
// them: it opens each file read-only and issues only SELECT. It knows nothing
// about Postgres; values come back as the SQLite driver's native types
// (int64, float64, string, []byte, nil) under their legacy column names.
//
// The package exists only for the import and is deleted with it.
package legacyread
import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"strings"
_ "modernc.org/sqlite"
)
// Source is one legacy SQLite file opened read-only. The process that owns
// the file must be stopped: a read-only open cannot checkpoint a WAL.
type Source struct {
db *sql.DB
}
// Open opens path read-only. It refuses a path that does not exist rather
// than letting SQLite create an empty database there.
func Open(ctx context.Context, path string) (*Source, error) {
if _, err := os.Stat(path); err != nil {
return nil, fmt.Errorf("legacyread: %w", err)
}
db, err := sql.Open("sqlite", "file:"+path+"?mode=ro&_pragma=busy_timeout(5000)")
if err != nil {
return nil, fmt.Errorf("legacyread: open %s: %w", path, err)
}
if err := db.PingContext(ctx); err != nil {
return nil, errors.Join(fmt.Errorf("legacyread: open %s: %w", path, err), db.Close())
}
return &Source{db: db}, nil
}
// Close closes the source.
func (s *Source) Close() error { return s.db.Close() }
// Table names one legacy table: its columns as the production code leaves
// them (columns added by ALTER TABLE included), the columns that identify a
// row, and the order it is read in.
type Table struct {
Name string
Columns []string
Key []string
OrderBy string
}
// Rows is one table's contents in read order.
type Rows struct {
Table Table
Rows [][]any
}
// Value returns column col of row i, or nil for an unknown column.
func (r Rows) Value(i int, col string) any {
for j, c := range r.Table.Columns {
if c == col {
return r.Rows[i][j]
}
}
return nil
}
// Keys returns each row's key, its key columns joined with "/", in read order.
func (r Rows) Keys() []string {
keys := make([]string, len(r.Rows))
for i := range r.Rows {
parts := make([]string, len(r.Table.Key))
for j, col := range r.Table.Key {
parts[j] = fmt.Sprint(r.Value(i, col))
}
keys[i] = strings.Join(parts, "/")
}
return keys
}
// Read returns every row of t.
func (s *Source) Read(ctx context.Context, t Table) (Rows, error) {
query := "SELECT " + strings.Join(t.Columns, ", ") + " FROM " + t.Name + " ORDER BY " + t.OrderBy
rows, err := s.db.QueryContext(ctx, query)
if err != nil {
return Rows{}, fmt.Errorf("legacyread: read %s: %w", t.Name, err)
}
defer func() { _ = rows.Close() }()
out := Rows{Table: t}
for rows.Next() {
row := make([]any, len(t.Columns))
ptrs := make([]any, len(row))
for i := range row {
ptrs[i] = &row[i]
}
if err := rows.Scan(ptrs...); err != nil {
return Rows{}, fmt.Errorf("legacyread: read %s: %w", t.Name, err)
}
out.Rows = append(out.Rows, row)
}
if err := rows.Err(); err != nil {
return Rows{}, fmt.Errorf("legacyread: read %s: %w", t.Name, err)
}
return out, nil
}
// IndexedMessageIDs returns the message ids the Gateway's FTS5 search index
// holds. messages_fts is an external-content index derived from messages, so
// it is rebuilt on the target rather than copied; what the import must account
// for is whether it covered every message, which comparing this set against
// the messages ids answers.
func (s *Source) IndexedMessageIDs(ctx context.Context) ([]string, error) {
return s.strings(ctx, "SELECT CAST(id AS TEXT) FROM messages_fts_docsize ORDER BY id")
}
func (s *Source) strings(ctx context.Context, query string) ([]string, error) {
rows, err := s.db.QueryContext(ctx, query)
if err != nil {
return nil, fmt.Errorf("legacyread: %w", err)
}
defer func() { _ = rows.Close() }()
var out []string
for rows.Next() {
var v string
if err := rows.Scan(&v); err != nil {
return nil, fmt.Errorf("legacyread: %w", err)
}
out = append(out, v)
}
return out, rows.Err()
}
@@ -0,0 +1,449 @@
package legacyread
import (
"crypto/sha256"
"database/sql"
"os"
"path/filepath"
"slices"
"testing"
)
// The fixtures below apply the legacy schemas as their production openers
// leave them, columns added by ALTER TABLE included, without importing those
// openers.
const stateStoreDDL = `
CREATE TABLE tasks (id INTEGER PRIMARY KEY AUTOINCREMENT, owner TEXT NOT NULL, repo TEXT NOT NULL,
issue_number INTEGER NOT NULL, title TEXT DEFAULT '', body TEXT DEFAULT '', labels TEXT DEFAULT '',
status TEXT DEFAULT 'queued', workflow TEXT DEFAULT '', stage TEXT DEFAULT '', branch TEXT DEFAULT '',
plan TEXT DEFAULT '', notes TEXT DEFAULT '', pr_number INTEGER DEFAULT 0, tokens_used INTEGER DEFAULT 0,
iterations INTEGER DEFAULT 0, attempt INTEGER DEFAULT 0, park_reason TEXT DEFAULT '',
watch_comment_id INTEGER DEFAULT 0, park_class TEXT DEFAULT 'needs_human', remediation_rounds INTEGER DEFAULT 0,
retry_count INTEGER DEFAULT 0, source TEXT DEFAULT 'forge', identity TEXT DEFAULT '', binding_id TEXT DEFAULT '',
binding_version INTEGER DEFAULT 0, review_payload TEXT DEFAULT '', workflow_definition_version INTEGER DEFAULT 0,
workflow_definition_digest TEXT DEFAULT '', workflow_definition_yaml TEXT DEFAULT '',
created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')));
ALTER TABLE tasks ADD COLUMN review_cursor INTEGER NOT NULL DEFAULT 0;
CREATE TABLE transitions (id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL,
at TEXT DEFAULT (datetime('now')), from_status TEXT NOT NULL, to_status TEXT NOT NULL, detail TEXT DEFAULT '');
CREATE TABLE events (id INTEGER PRIMARY KEY AUTOINCREMENT, at TEXT NOT NULL, kind TEXT NOT NULL,
task_id INTEGER DEFAULT 0, repo TEXT DEFAULT '', issue INTEGER DEFAULT 0, workflow TEXT DEFAULT '',
stage TEXT DEFAULT '', attempt INTEGER DEFAULT 0, actor_id TEXT DEFAULT '', actor_kind TEXT DEFAULT '',
principal_id TEXT DEFAULT '', detail TEXT DEFAULT '', data TEXT DEFAULT '{}');
CREATE TABLE resources (kind TEXT PRIMARY KEY, value BLOB NOT NULL, version INTEGER NOT NULL, updated_at TEXT NOT NULL);
CREATE TABLE resource_history (id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, value BLOB NOT NULL,
version INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL, request_id TEXT NOT NULL,
expected_version INTEGER NOT NULL, current_version INTEGER NOT NULL, at TEXT NOT NULL);
CREATE TABLE channel_status (id TEXT PRIMARY KEY, name TEXT DEFAULT '', state TEXT DEFAULT '', detail TEXT DEFAULT '',
configured INTEGER NOT NULL DEFAULT 0, reload_supported INTEGER NOT NULL DEFAULT 0, observed_at TEXT NOT NULL);
CREATE TABLE apply_status (process TEXT NOT NULL, kind TEXT NOT NULL, applied_version INTEGER DEFAULT 0,
error TEXT DEFAULT '', reported_at TEXT NOT NULL, PRIMARY KEY (process, kind));
CREATE TABLE config_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), schema TEXT DEFAULT '', document TEXT DEFAULT '',
published_at TEXT NOT NULL);
CREATE TABLE identities (id TEXT PRIMARY KEY, kind TEXT NOT NULL, display_name TEXT NOT NULL, lifecycle TEXT NOT NULL,
version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL);
CREATE TABLE identity_aliases (alias TEXT PRIMARY KEY COLLATE NOCASE, identity_id TEXT NOT NULL REFERENCES identities(id));
CREATE TABLE identity_events (id INTEGER PRIMARY KEY AUTOINCREMENT, identity_id TEXT NOT NULL, event_type TEXT NOT NULL,
from_lifecycle TEXT NOT NULL, to_lifecycle TEXT NOT NULL, display_name TEXT NOT NULL, actor_id TEXT NOT NULL,
source TEXT NOT NULL, request_id TEXT NOT NULL UNIQUE, at TEXT NOT NULL);
CREATE TABLE identity_subjects (issuer TEXT NOT NULL, subject TEXT NOT NULL,
identity_id TEXT NOT NULL REFERENCES identities(id), bound_at TEXT NOT NULL, PRIMARY KEY (issuer, subject));
INSERT INTO tasks (id, owner, repo, issue_number, review_cursor) VALUES (1, 'o', 'r', 7, 3), (4, 'o', 'r', 1000000000000000, 0);
INSERT INTO transitions (task_id, at, from_status, to_status) VALUES (1, '2026-01-01 00:00:00', 'queued', 'running');
INSERT INTO events (id, at, kind) VALUES
(1, '2026-01-01T00:00:00.000000100Z', 'a'),
(2, '2026-01-01T00:00:01.000000000Z', 'b');
INSERT INTO resources VALUES ('config', X'00ff10', 2, '2026-01-01T00:00:00Z');
INSERT INTO resource_history (kind, value, version, actor, source, request_id, expected_version, current_version, at)
VALUES ('config', X'00ff10', 2, 'sam', 'ui', 'req-1', 1, 1, '2026-01-01T00:00:00Z');
INSERT INTO channel_status VALUES ('tg', 'Telegram', 'up', '', 1, 0, '2026-01-01T00:00:00Z');
INSERT INTO apply_status VALUES ('daemon', 'config', 2, '', '2026-01-01T00:00:00Z');
INSERT INTO config_snapshot VALUES (1, 's', '{}', '2026-01-01T00:00:00Z');
INSERT INTO identities VALUES ('id-1', 'agent', 'Archie', 'active', 1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z');
INSERT INTO identity_aliases VALUES ('archie', 'id-1');
INSERT INTO identity_events (identity_id, event_type, from_lifecycle, to_lifecycle, display_name, actor_id, source, request_id, at)
VALUES ('id-1', 'created', '', 'active', 'Archie', 'sam', 'boot', 'rq-1', '2026-01-01T00:00:00Z');
INSERT INTO identity_subjects VALUES ('https://issuer', 'sub-1', 'id-1', '2026-01-01T00:00:00Z');
`
const edaDDL = `
CREATE TABLE captures (id TEXT PRIMARY KEY NOT NULL, authenticated BOOLEAN DEFAULT FALSE NOT NULL,
body TEXT DEFAULT '' NOT NULL, content_type TEXT DEFAULT '' NOT NULL, headers JSON DEFAULT NULL,
received_at TEXT DEFAULT '' NOT NULL, remote_addr TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL);
CREATE TABLE mappings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, fields JSON DEFAULT NULL,
name TEXT DEFAULT '' NOT NULL, source_hint TEXT DEFAULT '' NOT NULL, updated_at TEXT DEFAULT '' NOT NULL);
CREATE TABLE bindings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, mapping TEXT DEFAULT '' NOT NULL,
name TEXT DEFAULT '' NOT NULL, owner TEXT DEFAULT '' NOT NULL, repo TEXT DEFAULT '' NOT NULL,
secret TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL, status TEXT DEFAULT '' NOT NULL,
updated_at TEXT DEFAULT '' NOT NULL, version NUMERIC DEFAULT 0 NOT NULL, workflow TEXT DEFAULT '' NOT NULL);
CREATE TABLE binding_dispatches (id TEXT PRIMARY KEY NOT NULL, binding TEXT DEFAULT '' NOT NULL,
binding_version NUMERIC DEFAULT 0 NOT NULL, capture TEXT DEFAULT '' NOT NULL, dispatched_at TEXT DEFAULT '' NOT NULL,
task_id NUMERIC DEFAULT 0 NOT NULL);
CREATE UNIQUE INDEX idx_binding_dispatch_once ON binding_dispatches (binding, capture);
CREATE TABLE playbook_dispatches (id TEXT PRIMARY KEY NOT NULL, action_id TEXT DEFAULT '' NOT NULL,
dispatched_at TEXT DEFAULT '' NOT NULL, event_id TEXT DEFAULT '' NOT NULL, playbook_id TEXT DEFAULT '' NOT NULL,
playbook_version TEXT DEFAULT '' NOT NULL);
CREATE TABLE tool_calls (id TEXT PRIMARY KEY NOT NULL, args JSON DEFAULT NULL, attempt NUMERIC DEFAULT 0 NOT NULL,
called_at TEXT DEFAULT '' NOT NULL, duration_ms NUMERIC DEFAULT 0 NOT NULL, error TEXT DEFAULT '' NOT NULL,
result JSON DEFAULT NULL, task_id NUMERIC DEFAULT 0 NOT NULL, tool TEXT DEFAULT '' NOT NULL);
INSERT INTO captures VALUES ('rcap1', 1, '{}', 'application/json', '{"x":["1"]}', '2026-01-01T00:00:00.000000000Z', '127.0.0.1', 'gh');
INSERT INTO mappings VALUES ('rmap1', '2026-01-01', '[]', 'm', 'gh', '2026-01-01');
INSERT INTO bindings VALUES ('rbind1', '2026-01-01', 'rmap1', 'b', 'o', 'r', 'plaintext-secret', 'gh', 'armed', '2026-01-01', 1, 'implement');
INSERT INTO binding_dispatches VALUES ('rdisp1', 'rbind1', 1, 'rcap1', '2026-01-01', 4);
INSERT INTO playbook_dispatches VALUES ('rpb1', 'act', '2026-01-01', 'ev', 'pb', 'v1');
INSERT INTO tool_calls VALUES ('rtool1', '{}', 1, '2026-01-01', 12, '', '{}', 4, 'bash');
`
const gatewayDDL = `
CREATE TABLE sessions (session_id TEXT PRIMARY KEY, platform TEXT NOT NULL DEFAULT '', bot_user TEXT NOT NULL DEFAULT '',
channel_id TEXT NOT NULL DEFAULT '', thread_id TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '',
parent_session_id TEXT NOT NULL DEFAULT '', branch_name TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL DEFAULT 0, last_active_at INTEGER NOT NULL DEFAULT 0);
CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT NOT NULL, session_id TEXT NOT NULL,
source_id TEXT NOT NULL DEFAULT '', sender TEXT NOT NULL DEFAULT '', text TEXT NOT NULL DEFAULT '', ts INTEGER NOT NULL,
UNIQUE(session_id, message_id));
ALTER TABLE messages ADD COLUMN role TEXT NOT NULL DEFAULT '';
ALTER TABLE messages ADD COLUMN sender_id TEXT NOT NULL DEFAULT '';
CREATE VIRTUAL TABLE messages_fts USING fts5(sender, text, content='messages', content_rowid='id');
CREATE TRIGGER messages_ai AFTER INSERT ON messages BEGIN
INSERT INTO messages_fts(rowid, sender, text) VALUES (new.id, new.sender, new.text);
END;
CREATE TABLE turns (turn_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, source_id TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL, attempt INTEGER NOT NULL DEFAULT 0, owner_id TEXT NOT NULL DEFAULT '',
input_message_id TEXT NOT NULL DEFAULT '', assistant_message_id TEXT NOT NULL DEFAULT '',
partial_text TEXT NOT NULL DEFAULT '', response_text TEXT NOT NULL DEFAULT '', tool_calls TEXT NOT NULL DEFAULT '[]',
error TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL);
INSERT INTO sessions VALUES ('s1', 'telegram', 'bot', 'c1', '', 'Chat', '', '', 1000, 2000);
INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m1', 's1', 'sam', 'hello there', 1000, 'user');
INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m2', 's1', 'archie', 'hi', 1001, 'assistant');
INSERT INTO turns VALUES ('t1', 's1', '', 'done', 1, '', 'm1', 'm2', '', 'hi', '[]', '', 1000, 1001);
`
// fixture writes a legacy database built from ddl and returns its path. It
// is checkpointed into a single file so a hash covers all of its state.
func fixture(t *testing.T, name, ddl string) string {
t.Helper()
path := filepath.Join(t.TempDir(), name)
db, err := sql.Open("sqlite", "file:"+path)
if err != nil {
t.Fatalf("open fixture: %v", err)
}
defer func() { _ = db.Close() }()
if _, err := db.ExecContext(t.Context(), ddl); err != nil {
t.Fatalf("apply fixture DDL: %v", err)
}
return path
}
func exec(t *testing.T, path, stmt string) {
t.Helper()
db, err := sql.Open("sqlite", "file:"+path)
if err != nil {
t.Fatalf("open: %v", err)
}
defer func() { _ = db.Close() }()
if _, err := db.ExecContext(t.Context(), stmt); err != nil {
t.Fatalf("exec %q: %v", stmt, err)
}
}
func hashDir(t *testing.T, dir string) map[string][32]byte {
t.Helper()
sums := map[string][32]byte{}
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatalf("read dir: %v", err)
}
for _, e := range entries {
b, err := os.ReadFile(filepath.Join(dir, e.Name()))
if err != nil {
t.Fatalf("read %s: %v", e.Name(), err)
}
sums[e.Name()] = sha256.Sum256(b)
}
return sums
}
func openSource(t *testing.T, path string) *Source {
t.Helper()
src, err := Open(t.Context(), path)
if err != nil {
t.Fatalf("Open(%s): %v", path, err)
}
t.Cleanup(func() { _ = src.Close() })
return src
}
// TestReadersDoNotMutateSources reads every table of all three stores and
// asserts every file in each store's directory is byte-for-byte unchanged,
// including the FTS5 index read.
func TestReadersDoNotMutateSources(t *testing.T) {
stores := []struct {
name string
ddl string
tables []Table
rows map[string]int
}{
{"archie.db", stateStoreDDL, StateStoreTables, map[string]int{"tasks": 2, "events": 2, "identity_subjects": 1}},
{"archie.db-eda.sqlite", edaDDL, EDATables, map[string]int{"captures": 1, "bindings": 1, "tool_calls": 1}},
{"archie.db-conversations.sqlite", gatewayDDL, GatewayTables, map[string]int{"messages": 2, "turns": 1}},
}
for _, s := range stores {
t.Run(s.name, func(t *testing.T) {
path := fixture(t, s.name, s.ddl)
before := hashDir(t, filepath.Dir(path))
src := openSource(t, path)
for _, table := range s.tables {
rows, err := src.Read(t.Context(), table)
if err != nil {
t.Fatalf("Read(%s): %v", table.Name, err)
}
if want, ok := s.rows[table.Name]; ok && len(rows.Rows) != want {
t.Errorf("Read(%s) = %d rows, want %d", table.Name, len(rows.Rows), want)
}
}
if s.name == "archie.db-conversations.sqlite" {
if _, err := src.IndexedMessageIDs(t.Context()); err != nil {
t.Fatalf("IndexedMessageIDs: %v", err)
}
}
if err := src.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
if after := hashDir(t, filepath.Dir(path)); !mapsEqual(before, after) {
t.Fatalf("reading changed the source directory:\nbefore %x\nafter %x", before, after)
}
})
}
}
func mapsEqual(a, b map[string][32]byte) bool {
if len(a) != len(b) {
return false
}
for k, v := range a {
if b[k] != v {
return false
}
}
return true
}
// TestOpenRefusesWrites pins the read-only open: a write through the
// source's connection is an error, not a silent mutation.
func TestOpenRefusesWrites(t *testing.T) {
src := openSource(t, fixture(t, "archie.db", stateStoreDDL))
if _, err := src.db.ExecContext(t.Context(), "DELETE FROM tasks"); err == nil {
t.Fatal("DELETE through the read-only source succeeded")
}
}
func TestReadCarriesAlteredColumnsAndBlobs(t *testing.T) {
src := openSource(t, fixture(t, "archie.db", stateStoreDDL))
tasks, err := src.Read(t.Context(), tableNamed(t, StateStoreTables, "tasks"))
if err != nil {
t.Fatalf("Read(tasks): %v", err)
}
if got := tasks.Value(0, "review_cursor"); got != int64(3) {
t.Errorf("tasks[0].review_cursor = %#v, want 3 (an ALTER TABLE column)", got)
}
if got := tasks.Value(1, "issue_number"); got != int64(1_000_000_000_000_000) {
t.Errorf("tasks[1].issue_number = %#v, want the chat seed", got)
}
res, err := src.Read(t.Context(), tableNamed(t, StateStoreTables, "resources"))
if err != nil {
t.Fatalf("Read(resources): %v", err)
}
if got, ok := res.Value(0, "value").([]byte); !ok || string(got) != "\x00\xff\x10" {
t.Errorf("resources[0].value = %#v, want the raw bytes 00 ff 10", res.Value(0, "value"))
}
}
func tableNamed(t *testing.T, tables []Table, name string) Table {
t.Helper()
i := slices.IndexFunc(tables, func(tb Table) bool { return tb.Name == name })
if i < 0 {
t.Fatalf("no table %q", name)
}
return tables[i]
}
func TestCompareKeys(t *testing.T) {
tests := []struct {
name string
source []string
target []string
missing, extra []string
}{
{name: "equal with the same gap", source: []string{"1", "2", "5"}, target: []string{"5", "1", "2"}},
{name: "swallowed row", source: []string{"1", "2", "3"}, target: []string{"1", "3"}, missing: []string{"2"}},
{name: "same count, different ids", source: []string{"1", "2"}, target: []string{"1", "9"}, missing: []string{"2"}, extra: []string{"9"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := CompareKeys(tt.source, tt.target)
if !slices.Equal(got.Missing, tt.missing) || !slices.Equal(got.Extra, tt.extra) {
t.Errorf("CompareKeys = %+v, want missing %v extra %v", got, tt.missing, tt.extra)
}
})
}
}
func TestCompareOrder(t *testing.T) {
if got := CompareOrder([]int64{1, 2, 3}, []int64{1, 2, 3}); got != nil {
t.Errorf("CompareOrder(equal) = %+v, want nil", got)
}
got := CompareOrder([]int64{1, 2, 3}, []int64{1, 3, 2})
if got == nil || got.Position != 1 || got.Source != 2 || got.Target != 3 {
t.Errorf("CompareOrder(swapped) = %+v, want the first divergence at position 1 (2 vs 3)", got)
}
if got := CompareOrder([]int64{1, 2}, []int64{1}); got == nil || got.Position != 1 {
t.Errorf("CompareOrder(short target) = %+v, want a divergence at position 1", got)
}
}
// TestMicrosecondCollisions constructs D3.8's condition: two events whose
// instants differ only below a microsecond, with the later instant carrying
// the lower id, so Postgres's (at, id) order would reverse them.
func TestMicrosecondCollisions(t *testing.T) {
path := fixture(t, "archie.db", stateStoreDDL)
exec(t, path, `INSERT INTO events (id, at, kind) VALUES
(10, '2026-02-01T00:00:00.000000900Z', 'later instant, lower id'),
(11, '2026-02-01T00:00:00.000000100Z', 'earlier instant, higher id'),
(12, '2026-02-01T00:00:05.000000100Z', 'same microsecond, ids in order'),
(13, '2026-02-01T00:00:05.000000900Z', 'same microsecond, ids in order')`)
src := openSource(t, path)
got, err := src.MicrosecondCollisions(t.Context())
if err != nil {
t.Fatalf("MicrosecondCollisions: %v", err)
}
if len(got) != 1 || got[0].EarlierID != 11 || got[0].LaterID != 10 {
t.Fatalf("MicrosecondCollisions = %+v, want exactly the pair (11 before 10)", got)
}
}
func TestOrphans(t *testing.T) {
path := fixture(t, "archie.db", stateStoreDDL)
src := openSource(t, path)
for _, fk := range IdentityForeignKeys {
got, err := src.Orphans(t.Context(), fk)
if err != nil || len(got) != 0 {
t.Fatalf("Orphans(%s) on a clean fixture = %v, %v; want none", fk.Table, got, err)
}
}
_ = src.Close()
exec(t, path, `INSERT INTO identity_aliases VALUES ('ghost', 'id-missing')`)
src = openSource(t, path)
got, err := src.Orphans(t.Context(), IdentityForeignKeys[0])
if err != nil {
t.Fatalf("Orphans: %v", err)
}
if len(got) != 1 || got[0].Key != "ghost" || got[0].Parent != "id-missing" {
t.Fatalf("Orphans(identity_aliases) = %+v, want the ghost alias", got)
}
}
// TestDuplicateAliases pins the case-folding hazard: SQLite's NOCASE folds
// ASCII only, so two aliases differing in non-ASCII case coexist there and
// collide under Postgres's Unicode lower().
func TestDuplicateAliases(t *testing.T) {
path := fixture(t, "archie.db", stateStoreDDL)
exec(t, path, `INSERT INTO identity_aliases VALUES ('ÉMILE', 'id-1'), ('émile', 'id-1')`)
src := openSource(t, path)
got, err := src.DuplicateAliases(t.Context())
if err != nil {
t.Fatalf("DuplicateAliases: %v", err)
}
if len(got) != 1 || got[0].Key != "émile" || len(got[0].Values) != 2 {
t.Fatalf("DuplicateAliases = %+v, want one group of two for émile", got)
}
}
func TestDuplicateArmedBindings(t *testing.T) {
path := fixture(t, "archie.db-eda.sqlite", edaDDL)
src := openSource(t, path)
if got, err := src.DuplicateBindings(t.Context()); err != nil || len(got) != 0 {
t.Fatalf("DuplicateBindings on a clean fixture = %v, %v; want none", got, err)
}
_ = src.Close()
exec(t, path, `INSERT INTO bindings VALUES ('rbind2', '', 'rmap1', 'b2', 'o', 'r', '', 'gh', 'pending_approval', '', 1, 'implement')`)
src = openSource(t, path)
got, err := src.DuplicateBindings(t.Context())
if err != nil {
t.Fatalf("DuplicateBindings: %v", err)
}
if len(got) != 1 || got[0].Key != "gh" || !slices.Equal(got[0].Values, []string{"rbind1", "rbind2"}) {
t.Fatalf("DuplicateBindings = %+v, want source gh held by rbind1 and rbind2", got)
}
}
func TestSequences(t *testing.T) {
src := openSource(t, fixture(t, "archie.db", stateStoreDDL))
got, err := src.Sequences(t.Context())
if err != nil {
t.Fatalf("Sequences: %v", err)
}
want := map[string]Sequence{
"tasks": {Table: "tasks", MaxID: 4, Next: 5},
"transitions": {Table: "transitions", MaxID: 1, Next: 2},
"events": {Table: "events", MaxID: 2, Next: 3},
"resource_history": {Table: "resource_history", MaxID: 1, Next: 2},
"identity_events": {Table: "identity_events", MaxID: 1, Next: 2},
}
if len(got) != len(want) {
t.Fatalf("Sequences = %+v, want %d tables", got, len(want))
}
for _, s := range got {
if want[s.Table] != s {
t.Errorf("Sequences[%s] = %+v, want %+v", s.Table, s, want[s.Table])
}
}
}
// TestSequenceNextSkipsDeletedTop pins that Next comes from AUTOINCREMENT's
// high-water mark, not max(id): a deleted top row still reserves its id.
func TestSequenceNextSkipsDeletedTop(t *testing.T) {
path := fixture(t, "archie.db", stateStoreDDL)
exec(t, path, `DELETE FROM tasks WHERE id = 4`)
src := openSource(t, path)
got, err := src.Sequences(t.Context())
if err != nil {
t.Fatalf("Sequences: %v", err)
}
i := slices.IndexFunc(got, func(s Sequence) bool { return s.Table == "tasks" })
if got[i].MaxID != 1 || got[i].Next != 5 {
t.Fatalf("Sequences[tasks] = %+v, want max 1, next 5", got[i])
}
}
func TestCompareBlobs(t *testing.T) {
src := map[string][]byte{"config": {0, 0xff}, "routing": {1}}
if got := CompareBlobs(src, map[string][]byte{"config": {0, 0xff}, "routing": {1}}); len(got) != 0 {
t.Errorf("CompareBlobs(equal) = %v, want none", got)
}
got := CompareBlobs(src, map[string][]byte{"config": {0, 0xfe}})
if !slices.Equal(got, []string{"config", "routing"}) {
t.Errorf("CompareBlobs = %v, want config (changed) and routing (missing)", got)
}
}
func TestIndexedMessageIDsFindsAStaleIndex(t *testing.T) {
path := fixture(t, "archie.db-conversations.sqlite", gatewayDDL)
exec(t, path, `DROP TRIGGER messages_ai; INSERT INTO messages (message_id, session_id, text, ts) VALUES ('m3', 's1', 'unindexed', 1002)`)
src := openSource(t, path)
indexed, err := src.IndexedMessageIDs(t.Context())
if err != nil {
t.Fatalf("IndexedMessageIDs: %v", err)
}
messages, err := src.Read(t.Context(), tableNamed(t, GatewayTables, "messages"))
if err != nil {
t.Fatalf("Read(messages): %v", err)
}
got := CompareKeys(messages.Keys(), indexed)
if !slices.Equal(got.Missing, []string{"3"}) {
t.Fatalf("CompareKeys(messages, index) = %+v, want message 3 missing from the index", got)
}
}
@@ -0,0 +1,73 @@
package legacyread
func idTable(name string, columns ...string) Table {
return Table{Name: name, Columns: columns, Key: []string{columns[0]}, OrderBy: columns[0]}
}
// StateStoreTables are the twelve tables of the State Store's archie.db.
var StateStoreTables = []Table{
idTable("tasks", "id", "owner", "repo", "issue_number", "title", "body", "labels", "status",
"workflow", "stage", "branch", "plan", "notes", "pr_number", "tokens_used", "iterations",
"attempt", "park_reason", "watch_comment_id", "park_class", "remediation_rounds", "retry_count",
"source", "identity", "binding_id", "binding_version", "review_payload",
"workflow_definition_version", "workflow_definition_digest", "workflow_definition_yaml",
"created_at", "updated_at", "review_cursor"),
idTable("transitions", "id", "task_id", "at", "from_status", "to_status", "detail"),
{
Name: "events",
Columns: []string{
"id", "at", "kind", "task_id", "repo", "issue", "workflow", "stage", "attempt",
"actor_id", "actor_kind", "principal_id", "detail", "data",
},
Key: []string{"id"},
OrderBy: "at, id",
},
idTable("resources", "kind", "value", "version", "updated_at"),
idTable("resource_history", "id", "kind", "value", "version", "actor", "source", "request_id",
"expected_version", "current_version", "at"),
idTable("channel_status", "id", "name", "state", "detail", "configured", "reload_supported", "observed_at"),
{
Name: "apply_status",
Columns: []string{"process", "kind", "applied_version", "error", "reported_at"},
Key: []string{"process", "kind"},
OrderBy: "process, kind",
},
idTable("config_snapshot", "id", "schema", "document", "published_at"),
idTable("identities", "id", "kind", "display_name", "lifecycle", "version", "created_at", "updated_at"),
idTable("identity_aliases", "alias", "identity_id"),
idTable("identity_events", "id", "identity_id", "event_type", "from_lifecycle", "to_lifecycle",
"display_name", "actor_id", "source", "request_id", "at"),
{
Name: "identity_subjects",
Columns: []string{"issuer", "subject", "identity_id", "bound_at"},
Key: []string{"issuer", "subject"},
OrderBy: "issuer, subject",
},
}
// EDATables are the six app collections of the PocketBase event store.
// PocketBase's own tables (_collections, _params, _superusers, ...) are not
// archie data and are not read.
var EDATables = []Table{
idTable("captures", "id", "authenticated", "body", "content_type", "headers", "received_at",
"remote_addr", "source"),
idTable("mappings", "id", "created_at", "fields", "name", "source_hint", "updated_at"),
idTable("bindings", "id", "created_at", "mapping", "name", "owner", "repo", "secret", "source",
"status", "updated_at", "version", "workflow"),
idTable("binding_dispatches", "id", "binding", "binding_version", "capture", "dispatched_at", "task_id"),
idTable("playbook_dispatches", "id", "action_id", "dispatched_at", "event_id", "playbook_id",
"playbook_version"),
idTable("tool_calls", "id", "args", "attempt", "called_at", "duration_ms", "error", "result",
"task_id", "tool"),
}
// GatewayTables are the Gateway conversation store's three data tables. Its
// FTS5 index is read through IndexedMessageIDs.
var GatewayTables = []Table{
idTable("sessions", "session_id", "platform", "bot_user", "channel_id", "thread_id", "title",
"parent_session_id", "branch_name", "created_at", "last_active_at"),
idTable("messages", "id", "message_id", "session_id", "source_id", "sender", "text", "ts", "role", "sender_id"),
idTable("turns", "turn_id", "session_id", "source_id", "status", "attempt", "owner_id",
"input_message_id", "assistant_message_id", "partial_text", "response_text", "tool_calls",
"error", "created_at", "updated_at"),
}
@@ -0,0 +1,275 @@
package legacyread
import (
"bytes"
"context"
"fmt"
"maps"
"slices"
"strings"
"time"
)
// KeyDiff is the difference between two key sets. A count comparison alone
// passes on a swallowed row when ids have gaps; comparing the sets does not.
type KeyDiff struct {
Missing []string // in the source, not the target
Extra []string // in the target, not the source
}
// CompareKeys compares the source's key set with the target's.
func CompareKeys(source, target []string) KeyDiff {
var d KeyDiff
src := setOf(source)
tgt := setOf(target)
for k := range src {
if !tgt[k] {
d.Missing = append(d.Missing, k)
}
}
for k := range tgt {
if !src[k] {
d.Extra = append(d.Extra, k)
}
}
slices.Sort(d.Missing)
slices.Sort(d.Extra)
return d
}
func setOf(keys []string) map[string]bool {
set := make(map[string]bool, len(keys))
for _, k := range keys {
set[k] = true
}
return set
}
// Divergence is the first position at which two ordered id sequences differ.
// A missing element on one side reads as 0.
type Divergence struct {
Position int
Source, Target int64
}
// CompareOrder compares two id sequences read in cursor order, returning nil
// when they are identical.
func CompareOrder(source, target []int64) *Divergence {
for i := range max(len(source), len(target)) {
var s, t int64
if i < len(source) {
s = source[i]
}
if i < len(target) {
t = target[i]
}
if s != t || i >= len(source) || i >= len(target) {
return &Divergence{Position: i, Source: s, Target: t}
}
}
return nil
}
// Collision is two events whose instants differ only below a microsecond
// while their ids run the other way. Postgres stores microseconds, so its
// (at, id) order would put LaterID before EarlierID.
type Collision struct {
EarlierID, LaterID int64
At time.Time // the shared microsecond
}
// eventAtLayouts are the layouts events.at has been written in, newest first.
var eventAtLayouts = []string{"2006-01-02T15:04:05.000000000Z", time.RFC3339Nano, time.RFC3339}
func parseEventAt(s string) (time.Time, error) {
for _, layout := range eventAtLayouts {
if t, err := time.Parse(layout, s); err == nil {
return t.UTC(), nil
}
}
return time.Time{}, fmt.Errorf("unrecognised timestamp %q", s)
}
// MicrosecondCollisions returns every event pair that truncating events.at
// to microseconds would reorder.
func (s *Source) MicrosecondCollisions(ctx context.Context) ([]Collision, error) {
rows, err := s.db.QueryContext(ctx, "SELECT id, at FROM events ORDER BY id")
if err != nil {
return nil, fmt.Errorf("legacyread: events: %w", err)
}
defer func() { _ = rows.Close() }()
type event struct {
id int64
at time.Time
}
byMicro := map[time.Time][]event{}
for rows.Next() {
var (
id int64
raw string
)
if err := rows.Scan(&id, &raw); err != nil {
return nil, fmt.Errorf("legacyread: events: %w", err)
}
at, err := parseEventAt(raw)
if err != nil {
return nil, fmt.Errorf("legacyread: event %d: %w", id, err)
}
micro := at.Truncate(time.Microsecond)
byMicro[micro] = append(byMicro[micro], event{id, at})
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("legacyread: events: %w", err)
}
var out []Collision
for micro, events := range byMicro {
for i, a := range events {
for _, b := range events[i+1:] {
// Read in id order, so a.id < b.id.
if b.at.Before(a.at) {
out = append(out, Collision{EarlierID: b.id, LaterID: a.id, At: micro})
}
}
}
}
slices.SortFunc(out, func(x, y Collision) int { return int(x.EarlierID - y.EarlierID) })
return out, nil
}
// ForeignKey is a reference SQLite declared but never enforced (the State
// Store never set PRAGMA foreign_keys) and Postgres will.
type ForeignKey struct {
Table, Column string
KeyExpr string // SQL naming the child row in a report
Parent, ParentC string
}
// IdentityForeignKeys are the three identity references.
var IdentityForeignKeys = []ForeignKey{
{Table: "identity_aliases", Column: "identity_id", KeyExpr: "alias", Parent: "identities", ParentC: "id"},
{Table: "identity_events", Column: "identity_id", KeyExpr: "CAST(id AS TEXT)", Parent: "identities", ParentC: "id"},
{Table: "identity_subjects", Column: "identity_id", KeyExpr: "issuer || '/' || subject", Parent: "identities", ParentC: "id"},
}
// Orphan is a child row whose parent does not exist.
type Orphan struct {
Key, Parent string
}
// Orphans returns fk's child rows with no parent.
func (s *Source) Orphans(ctx context.Context, fk ForeignKey) ([]Orphan, error) {
query := fmt.Sprintf("SELECT %s, %s FROM %s WHERE %s NOT IN (SELECT %s FROM %s) ORDER BY 1",
fk.KeyExpr, fk.Column, fk.Table, fk.Column, fk.ParentC, fk.Parent)
rows, err := s.db.QueryContext(ctx, query)
if err != nil {
return nil, fmt.Errorf("legacyread: orphans in %s: %w", fk.Table, err)
}
defer func() { _ = rows.Close() }()
var out []Orphan
for rows.Next() {
var o Orphan
if err := rows.Scan(&o.Key, &o.Parent); err != nil {
return nil, fmt.Errorf("legacyread: orphans in %s: %w", fk.Table, err)
}
out = append(out, o)
}
return out, rows.Err()
}
// Duplicate is a key the target will hold unique that more than one source
// row shares.
type Duplicate struct {
Key string
Values []string
}
// DuplicateAliases groups identity aliases by Unicode lower case. SQLite's
// NOCASE folds ASCII only, so aliases that differ in non-ASCII case coexist in
// the source and collide on the target's lower(alias) index.
func (s *Source) DuplicateAliases(ctx context.Context) ([]Duplicate, error) {
aliases, err := s.strings(ctx, "SELECT alias FROM identity_aliases ORDER BY alias")
if err != nil {
return nil, err
}
return duplicates(aliases, aliases, strings.ToLower), nil
}
// DuplicateBindings groups bindings by source. One binding per source is
// enforced in Go, not by the legacy schema, so a duplicate is possible and is
// refused rather than repaired (D4.2).
func (s *Source) DuplicateBindings(ctx context.Context) ([]Duplicate, error) {
rows, err := s.Read(ctx, Table{Name: "bindings", Columns: []string{"id", "source"}, OrderBy: "id"})
if err != nil {
return nil, err
}
ids := make([]string, len(rows.Rows))
sources := make([]string, len(rows.Rows))
for i := range rows.Rows {
ids[i] = fmt.Sprint(rows.Value(i, "id"))
sources[i] = fmt.Sprint(rows.Value(i, "source"))
}
return duplicates(sources, ids, func(s string) string { return s }), nil
}
// duplicates groups values by key(keys[i]) and returns the groups of two or
// more, sorted by key.
func duplicates(keys, values []string, key func(string) string) []Duplicate {
groups := map[string][]string{}
for i, k := range keys {
groups[key(k)] = append(groups[key(k)], values[i])
}
var out []Duplicate
for _, k := range slices.Sorted(maps.Keys(groups)) {
if len(groups[k]) > 1 {
out = append(out, Duplicate{Key: k, Values: groups[k]})
}
}
return out
}
// Sequence is an identity column's state: the highest id present and the id
// SQLite would assign next. Next comes from AUTOINCREMENT's high-water mark,
// so a deleted top row still reserves its id.
type Sequence struct {
Table string
MaxID, Next int64
}
// sequenceTables are the State Store tables with an AUTOINCREMENT id.
var sequenceTables = []string{"tasks", "transitions", "events", "resource_history", "identity_events"}
// Sequences returns the state of every State Store identity column.
func (s *Source) Sequences(ctx context.Context) ([]Sequence, error) {
out := make([]Sequence, 0, len(sequenceTables))
for _, table := range sequenceTables {
seq := Sequence{Table: table}
var high int64
err := s.db.QueryRowContext(ctx, fmt.Sprintf(
"SELECT COALESCE((SELECT max(id) FROM %[1]s), 0), COALESCE((SELECT seq FROM sqlite_sequence WHERE name = '%[1]s'), 0)",
table)).Scan(&seq.MaxID, &high)
if err != nil {
return nil, fmt.Errorf("legacyread: sequence %s: %w", table, err)
}
seq.Next = max(seq.MaxID, high) + 1
out = append(out, seq)
}
return out, nil
}
// CompareBlobs returns, sorted, every key whose bytes differ between source
// and target or that only one side holds.
func CompareBlobs(source, target map[string][]byte) []string {
var out []string
for k, v := range source {
if t, ok := target[k]; !ok || !bytes.Equal(v, t) {
out = append(out, k)
}
}
for k := range target {
if _, ok := source[k]; !ok {
out = append(out, k)
}
}
slices.Sort(out)
return out
}