feat(legacyread): read-only readers and import verification for the legacy SQLite stores
This commit is contained in:
4 files changed
+936
No files matched your search
@@ -0,0 +1,139 @@
|
||||
// Package legacyread reads an existing install's SQLite stores (the State
|
||||
// Store's archie.db, the PocketBase event store and the Gateway conversation
|
||||
// store) for the one-time import into Postgres, and verifies what was read.
|
||||
//
|
||||
// Every production opener of those files writes on open (schema migrations,
|
||||
// PocketBase bootstrap, DDL and backfills), so this package never imports
|
||||
// them: it opens each file read-only and issues only SELECT. It knows nothing
|
||||
// about Postgres; values come back as the SQLite driver's native types
|
||||
// (int64, float64, string, []byte, nil) under their legacy column names.
|
||||
//
|
||||
// The package exists only for the import and is deleted with it.
|
||||
package legacyread
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// Source is one legacy SQLite file opened read-only. The process that owns
|
||||
// the file must be stopped: a read-only open cannot checkpoint a WAL.
|
||||
type Source struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// Open opens path read-only. It refuses a path that does not exist rather
|
||||
// than letting SQLite create an empty database there.
|
||||
func Open(ctx context.Context, path string) (*Source, error) {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return nil, fmt.Errorf("legacyread: %w", err)
|
||||
}
|
||||
db, err := sql.Open("sqlite", "file:"+path+"?mode=ro&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacyread: open %s: %w", path, err)
|
||||
}
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("legacyread: open %s: %w", path, err), db.Close())
|
||||
}
|
||||
return &Source{db: db}, nil
|
||||
}
|
||||
|
||||
// Close closes the source.
|
||||
func (s *Source) Close() error { return s.db.Close() }
|
||||
|
||||
// Table names one legacy table: its columns as the production code leaves
|
||||
// them (columns added by ALTER TABLE included), the columns that identify a
|
||||
// row, and the order it is read in.
|
||||
type Table struct {
|
||||
Name string
|
||||
Columns []string
|
||||
Key []string
|
||||
OrderBy string
|
||||
}
|
||||
|
||||
// Rows is one table's contents in read order.
|
||||
type Rows struct {
|
||||
Table Table
|
||||
Rows [][]any
|
||||
}
|
||||
|
||||
// Value returns column col of row i, or nil for an unknown column.
|
||||
func (r Rows) Value(i int, col string) any {
|
||||
for j, c := range r.Table.Columns {
|
||||
if c == col {
|
||||
return r.Rows[i][j]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Keys returns each row's key, its key columns joined with "/", in read order.
|
||||
func (r Rows) Keys() []string {
|
||||
keys := make([]string, len(r.Rows))
|
||||
for i := range r.Rows {
|
||||
parts := make([]string, len(r.Table.Key))
|
||||
for j, col := range r.Table.Key {
|
||||
parts[j] = fmt.Sprint(r.Value(i, col))
|
||||
}
|
||||
keys[i] = strings.Join(parts, "/")
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// Read returns every row of t.
|
||||
func (s *Source) Read(ctx context.Context, t Table) (Rows, error) {
|
||||
query := "SELECT " + strings.Join(t.Columns, ", ") + " FROM " + t.Name + " ORDER BY " + t.OrderBy
|
||||
rows, err := s.db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return Rows{}, fmt.Errorf("legacyread: read %s: %w", t.Name, err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
out := Rows{Table: t}
|
||||
for rows.Next() {
|
||||
row := make([]any, len(t.Columns))
|
||||
ptrs := make([]any, len(row))
|
||||
for i := range row {
|
||||
ptrs[i] = &row[i]
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
return Rows{}, fmt.Errorf("legacyread: read %s: %w", t.Name, err)
|
||||
}
|
||||
out.Rows = append(out.Rows, row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return Rows{}, fmt.Errorf("legacyread: read %s: %w", t.Name, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// IndexedMessageIDs returns the message ids the Gateway's FTS5 search index
|
||||
// holds. messages_fts is an external-content index derived from messages, so
|
||||
// it is rebuilt on the target rather than copied; what the import must account
|
||||
// for is whether it covered every message, which comparing this set against
|
||||
// the messages ids answers.
|
||||
func (s *Source) IndexedMessageIDs(ctx context.Context) ([]string, error) {
|
||||
return s.strings(ctx, "SELECT CAST(id AS TEXT) FROM messages_fts_docsize ORDER BY id")
|
||||
}
|
||||
|
||||
func (s *Source) strings(ctx context.Context, query string) ([]string, error) {
|
||||
rows, err := s.db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacyread: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var v string
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
return nil, fmt.Errorf("legacyread: %w", err)
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,449 @@
|
||||
package legacyread
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The fixtures below apply the legacy schemas as their production openers
|
||||
// leave them, columns added by ALTER TABLE included, without importing those
|
||||
// openers.
|
||||
const stateStoreDDL = `
|
||||
CREATE TABLE tasks (id INTEGER PRIMARY KEY AUTOINCREMENT, owner TEXT NOT NULL, repo TEXT NOT NULL,
|
||||
issue_number INTEGER NOT NULL, title TEXT DEFAULT '', body TEXT DEFAULT '', labels TEXT DEFAULT '',
|
||||
status TEXT DEFAULT 'queued', workflow TEXT DEFAULT '', stage TEXT DEFAULT '', branch TEXT DEFAULT '',
|
||||
plan TEXT DEFAULT '', notes TEXT DEFAULT '', pr_number INTEGER DEFAULT 0, tokens_used INTEGER DEFAULT 0,
|
||||
iterations INTEGER DEFAULT 0, attempt INTEGER DEFAULT 0, park_reason TEXT DEFAULT '',
|
||||
watch_comment_id INTEGER DEFAULT 0, park_class TEXT DEFAULT 'needs_human', remediation_rounds INTEGER DEFAULT 0,
|
||||
retry_count INTEGER DEFAULT 0, source TEXT DEFAULT 'forge', identity TEXT DEFAULT '', binding_id TEXT DEFAULT '',
|
||||
binding_version INTEGER DEFAULT 0, review_payload TEXT DEFAULT '', workflow_definition_version INTEGER DEFAULT 0,
|
||||
workflow_definition_digest TEXT DEFAULT '', workflow_definition_yaml TEXT DEFAULT '',
|
||||
created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')));
|
||||
ALTER TABLE tasks ADD COLUMN review_cursor INTEGER NOT NULL DEFAULT 0;
|
||||
CREATE TABLE transitions (id INTEGER PRIMARY KEY AUTOINCREMENT, task_id INTEGER NOT NULL,
|
||||
at TEXT DEFAULT (datetime('now')), from_status TEXT NOT NULL, to_status TEXT NOT NULL, detail TEXT DEFAULT '');
|
||||
CREATE TABLE events (id INTEGER PRIMARY KEY AUTOINCREMENT, at TEXT NOT NULL, kind TEXT NOT NULL,
|
||||
task_id INTEGER DEFAULT 0, repo TEXT DEFAULT '', issue INTEGER DEFAULT 0, workflow TEXT DEFAULT '',
|
||||
stage TEXT DEFAULT '', attempt INTEGER DEFAULT 0, actor_id TEXT DEFAULT '', actor_kind TEXT DEFAULT '',
|
||||
principal_id TEXT DEFAULT '', detail TEXT DEFAULT '', data TEXT DEFAULT '{}');
|
||||
CREATE TABLE resources (kind TEXT PRIMARY KEY, value BLOB NOT NULL, version INTEGER NOT NULL, updated_at TEXT NOT NULL);
|
||||
CREATE TABLE resource_history (id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, value BLOB NOT NULL,
|
||||
version INTEGER NOT NULL, actor TEXT NOT NULL, source TEXT NOT NULL, request_id TEXT NOT NULL,
|
||||
expected_version INTEGER NOT NULL, current_version INTEGER NOT NULL, at TEXT NOT NULL);
|
||||
CREATE TABLE channel_status (id TEXT PRIMARY KEY, name TEXT DEFAULT '', state TEXT DEFAULT '', detail TEXT DEFAULT '',
|
||||
configured INTEGER NOT NULL DEFAULT 0, reload_supported INTEGER NOT NULL DEFAULT 0, observed_at TEXT NOT NULL);
|
||||
CREATE TABLE apply_status (process TEXT NOT NULL, kind TEXT NOT NULL, applied_version INTEGER DEFAULT 0,
|
||||
error TEXT DEFAULT '', reported_at TEXT NOT NULL, PRIMARY KEY (process, kind));
|
||||
CREATE TABLE config_snapshot (id INTEGER PRIMARY KEY CHECK (id = 1), schema TEXT DEFAULT '', document TEXT DEFAULT '',
|
||||
published_at TEXT NOT NULL);
|
||||
CREATE TABLE identities (id TEXT PRIMARY KEY, kind TEXT NOT NULL, display_name TEXT NOT NULL, lifecycle TEXT NOT NULL,
|
||||
version INTEGER NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL);
|
||||
CREATE TABLE identity_aliases (alias TEXT PRIMARY KEY COLLATE NOCASE, identity_id TEXT NOT NULL REFERENCES identities(id));
|
||||
CREATE TABLE identity_events (id INTEGER PRIMARY KEY AUTOINCREMENT, identity_id TEXT NOT NULL, event_type TEXT NOT NULL,
|
||||
from_lifecycle TEXT NOT NULL, to_lifecycle TEXT NOT NULL, display_name TEXT NOT NULL, actor_id TEXT NOT NULL,
|
||||
source TEXT NOT NULL, request_id TEXT NOT NULL UNIQUE, at TEXT NOT NULL);
|
||||
CREATE TABLE identity_subjects (issuer TEXT NOT NULL, subject TEXT NOT NULL,
|
||||
identity_id TEXT NOT NULL REFERENCES identities(id), bound_at TEXT NOT NULL, PRIMARY KEY (issuer, subject));
|
||||
|
||||
INSERT INTO tasks (id, owner, repo, issue_number, review_cursor) VALUES (1, 'o', 'r', 7, 3), (4, 'o', 'r', 1000000000000000, 0);
|
||||
INSERT INTO transitions (task_id, at, from_status, to_status) VALUES (1, '2026-01-01 00:00:00', 'queued', 'running');
|
||||
INSERT INTO events (id, at, kind) VALUES
|
||||
(1, '2026-01-01T00:00:00.000000100Z', 'a'),
|
||||
(2, '2026-01-01T00:00:01.000000000Z', 'b');
|
||||
INSERT INTO resources VALUES ('config', X'00ff10', 2, '2026-01-01T00:00:00Z');
|
||||
INSERT INTO resource_history (kind, value, version, actor, source, request_id, expected_version, current_version, at)
|
||||
VALUES ('config', X'00ff10', 2, 'sam', 'ui', 'req-1', 1, 1, '2026-01-01T00:00:00Z');
|
||||
INSERT INTO channel_status VALUES ('tg', 'Telegram', 'up', '', 1, 0, '2026-01-01T00:00:00Z');
|
||||
INSERT INTO apply_status VALUES ('daemon', 'config', 2, '', '2026-01-01T00:00:00Z');
|
||||
INSERT INTO config_snapshot VALUES (1, 's', '{}', '2026-01-01T00:00:00Z');
|
||||
INSERT INTO identities VALUES ('id-1', 'agent', 'Archie', 'active', 1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z');
|
||||
INSERT INTO identity_aliases VALUES ('archie', 'id-1');
|
||||
INSERT INTO identity_events (identity_id, event_type, from_lifecycle, to_lifecycle, display_name, actor_id, source, request_id, at)
|
||||
VALUES ('id-1', 'created', '', 'active', 'Archie', 'sam', 'boot', 'rq-1', '2026-01-01T00:00:00Z');
|
||||
INSERT INTO identity_subjects VALUES ('https://issuer', 'sub-1', 'id-1', '2026-01-01T00:00:00Z');
|
||||
`
|
||||
|
||||
const edaDDL = `
|
||||
CREATE TABLE captures (id TEXT PRIMARY KEY NOT NULL, authenticated BOOLEAN DEFAULT FALSE NOT NULL,
|
||||
body TEXT DEFAULT '' NOT NULL, content_type TEXT DEFAULT '' NOT NULL, headers JSON DEFAULT NULL,
|
||||
received_at TEXT DEFAULT '' NOT NULL, remote_addr TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL);
|
||||
CREATE TABLE mappings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, fields JSON DEFAULT NULL,
|
||||
name TEXT DEFAULT '' NOT NULL, source_hint TEXT DEFAULT '' NOT NULL, updated_at TEXT DEFAULT '' NOT NULL);
|
||||
CREATE TABLE bindings (id TEXT PRIMARY KEY NOT NULL, created_at TEXT DEFAULT '' NOT NULL, mapping TEXT DEFAULT '' NOT NULL,
|
||||
name TEXT DEFAULT '' NOT NULL, owner TEXT DEFAULT '' NOT NULL, repo TEXT DEFAULT '' NOT NULL,
|
||||
secret TEXT DEFAULT '' NOT NULL, source TEXT DEFAULT '' NOT NULL, status TEXT DEFAULT '' NOT NULL,
|
||||
updated_at TEXT DEFAULT '' NOT NULL, version NUMERIC DEFAULT 0 NOT NULL, workflow TEXT DEFAULT '' NOT NULL);
|
||||
CREATE TABLE binding_dispatches (id TEXT PRIMARY KEY NOT NULL, binding TEXT DEFAULT '' NOT NULL,
|
||||
binding_version NUMERIC DEFAULT 0 NOT NULL, capture TEXT DEFAULT '' NOT NULL, dispatched_at TEXT DEFAULT '' NOT NULL,
|
||||
task_id NUMERIC DEFAULT 0 NOT NULL);
|
||||
CREATE UNIQUE INDEX idx_binding_dispatch_once ON binding_dispatches (binding, capture);
|
||||
CREATE TABLE playbook_dispatches (id TEXT PRIMARY KEY NOT NULL, action_id TEXT DEFAULT '' NOT NULL,
|
||||
dispatched_at TEXT DEFAULT '' NOT NULL, event_id TEXT DEFAULT '' NOT NULL, playbook_id TEXT DEFAULT '' NOT NULL,
|
||||
playbook_version TEXT DEFAULT '' NOT NULL);
|
||||
CREATE TABLE tool_calls (id TEXT PRIMARY KEY NOT NULL, args JSON DEFAULT NULL, attempt NUMERIC DEFAULT 0 NOT NULL,
|
||||
called_at TEXT DEFAULT '' NOT NULL, duration_ms NUMERIC DEFAULT 0 NOT NULL, error TEXT DEFAULT '' NOT NULL,
|
||||
result JSON DEFAULT NULL, task_id NUMERIC DEFAULT 0 NOT NULL, tool TEXT DEFAULT '' NOT NULL);
|
||||
|
||||
INSERT INTO captures VALUES ('rcap1', 1, '{}', 'application/json', '{"x":["1"]}', '2026-01-01T00:00:00.000000000Z', '127.0.0.1', 'gh');
|
||||
INSERT INTO mappings VALUES ('rmap1', '2026-01-01', '[]', 'm', 'gh', '2026-01-01');
|
||||
INSERT INTO bindings VALUES ('rbind1', '2026-01-01', 'rmap1', 'b', 'o', 'r', 'plaintext-secret', 'gh', 'armed', '2026-01-01', 1, 'implement');
|
||||
INSERT INTO binding_dispatches VALUES ('rdisp1', 'rbind1', 1, 'rcap1', '2026-01-01', 4);
|
||||
INSERT INTO playbook_dispatches VALUES ('rpb1', 'act', '2026-01-01', 'ev', 'pb', 'v1');
|
||||
INSERT INTO tool_calls VALUES ('rtool1', '{}', 1, '2026-01-01', 12, '', '{}', 4, 'bash');
|
||||
`
|
||||
|
||||
const gatewayDDL = `
|
||||
CREATE TABLE sessions (session_id TEXT PRIMARY KEY, platform TEXT NOT NULL DEFAULT '', bot_user TEXT NOT NULL DEFAULT '',
|
||||
channel_id TEXT NOT NULL DEFAULT '', thread_id TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '',
|
||||
parent_session_id TEXT NOT NULL DEFAULT '', branch_name TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL DEFAULT 0, last_active_at INTEGER NOT NULL DEFAULT 0);
|
||||
CREATE TABLE messages (id INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT NOT NULL, session_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL DEFAULT '', sender TEXT NOT NULL DEFAULT '', text TEXT NOT NULL DEFAULT '', ts INTEGER NOT NULL,
|
||||
UNIQUE(session_id, message_id));
|
||||
ALTER TABLE messages ADD COLUMN role TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE messages ADD COLUMN sender_id TEXT NOT NULL DEFAULT '';
|
||||
CREATE VIRTUAL TABLE messages_fts USING fts5(sender, text, content='messages', content_rowid='id');
|
||||
CREATE TRIGGER messages_ai AFTER INSERT ON messages BEGIN
|
||||
INSERT INTO messages_fts(rowid, sender, text) VALUES (new.id, new.sender, new.text);
|
||||
END;
|
||||
CREATE TABLE turns (turn_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, source_id TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL, attempt INTEGER NOT NULL DEFAULT 0, owner_id TEXT NOT NULL DEFAULT '',
|
||||
input_message_id TEXT NOT NULL DEFAULT '', assistant_message_id TEXT NOT NULL DEFAULT '',
|
||||
partial_text TEXT NOT NULL DEFAULT '', response_text TEXT NOT NULL DEFAULT '', tool_calls TEXT NOT NULL DEFAULT '[]',
|
||||
error TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL);
|
||||
|
||||
INSERT INTO sessions VALUES ('s1', 'telegram', 'bot', 'c1', '', 'Chat', '', '', 1000, 2000);
|
||||
INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m1', 's1', 'sam', 'hello there', 1000, 'user');
|
||||
INSERT INTO messages (message_id, session_id, sender, text, ts, role) VALUES ('m2', 's1', 'archie', 'hi', 1001, 'assistant');
|
||||
INSERT INTO turns VALUES ('t1', 's1', '', 'done', 1, '', 'm1', 'm2', '', 'hi', '[]', '', 1000, 1001);
|
||||
`
|
||||
|
||||
// fixture writes a legacy database built from ddl and returns its path. It
|
||||
// is checkpointed into a single file so a hash covers all of its state.
|
||||
func fixture(t *testing.T, name, ddl string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), name)
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
t.Fatalf("open fixture: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
if _, err := db.ExecContext(t.Context(), ddl); err != nil {
|
||||
t.Fatalf("apply fixture DDL: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func exec(t *testing.T, path, stmt string) {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
if _, err := db.ExecContext(t.Context(), stmt); err != nil {
|
||||
t.Fatalf("exec %q: %v", stmt, err)
|
||||
}
|
||||
}
|
||||
|
||||
func hashDir(t *testing.T, dir string) map[string][32]byte {
|
||||
t.Helper()
|
||||
sums := map[string][32]byte{}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read dir: %v", err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
b, err := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", e.Name(), err)
|
||||
}
|
||||
sums[e.Name()] = sha256.Sum256(b)
|
||||
}
|
||||
return sums
|
||||
}
|
||||
|
||||
func openSource(t *testing.T, path string) *Source {
|
||||
t.Helper()
|
||||
src, err := Open(t.Context(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("Open(%s): %v", path, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = src.Close() })
|
||||
return src
|
||||
}
|
||||
|
||||
// TestReadersDoNotMutateSources reads every table of all three stores and
|
||||
// asserts every file in each store's directory is byte-for-byte unchanged,
|
||||
// including the FTS5 index read.
|
||||
func TestReadersDoNotMutateSources(t *testing.T) {
|
||||
stores := []struct {
|
||||
name string
|
||||
ddl string
|
||||
tables []Table
|
||||
rows map[string]int
|
||||
}{
|
||||
{"archie.db", stateStoreDDL, StateStoreTables, map[string]int{"tasks": 2, "events": 2, "identity_subjects": 1}},
|
||||
{"archie.db-eda.sqlite", edaDDL, EDATables, map[string]int{"captures": 1, "bindings": 1, "tool_calls": 1}},
|
||||
{"archie.db-conversations.sqlite", gatewayDDL, GatewayTables, map[string]int{"messages": 2, "turns": 1}},
|
||||
}
|
||||
for _, s := range stores {
|
||||
t.Run(s.name, func(t *testing.T) {
|
||||
path := fixture(t, s.name, s.ddl)
|
||||
before := hashDir(t, filepath.Dir(path))
|
||||
src := openSource(t, path)
|
||||
for _, table := range s.tables {
|
||||
rows, err := src.Read(t.Context(), table)
|
||||
if err != nil {
|
||||
t.Fatalf("Read(%s): %v", table.Name, err)
|
||||
}
|
||||
if want, ok := s.rows[table.Name]; ok && len(rows.Rows) != want {
|
||||
t.Errorf("Read(%s) = %d rows, want %d", table.Name, len(rows.Rows), want)
|
||||
}
|
||||
}
|
||||
if s.name == "archie.db-conversations.sqlite" {
|
||||
if _, err := src.IndexedMessageIDs(t.Context()); err != nil {
|
||||
t.Fatalf("IndexedMessageIDs: %v", err)
|
||||
}
|
||||
}
|
||||
if err := src.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
if after := hashDir(t, filepath.Dir(path)); !mapsEqual(before, after) {
|
||||
t.Fatalf("reading changed the source directory:\nbefore %x\nafter %x", before, after)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mapsEqual(a, b map[string][32]byte) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for k, v := range a {
|
||||
if b[k] != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// TestOpenRefusesWrites pins the read-only open: a write through the
|
||||
// source's connection is an error, not a silent mutation.
|
||||
func TestOpenRefusesWrites(t *testing.T) {
|
||||
src := openSource(t, fixture(t, "archie.db", stateStoreDDL))
|
||||
if _, err := src.db.ExecContext(t.Context(), "DELETE FROM tasks"); err == nil {
|
||||
t.Fatal("DELETE through the read-only source succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadCarriesAlteredColumnsAndBlobs(t *testing.T) {
|
||||
src := openSource(t, fixture(t, "archie.db", stateStoreDDL))
|
||||
tasks, err := src.Read(t.Context(), tableNamed(t, StateStoreTables, "tasks"))
|
||||
if err != nil {
|
||||
t.Fatalf("Read(tasks): %v", err)
|
||||
}
|
||||
if got := tasks.Value(0, "review_cursor"); got != int64(3) {
|
||||
t.Errorf("tasks[0].review_cursor = %#v, want 3 (an ALTER TABLE column)", got)
|
||||
}
|
||||
if got := tasks.Value(1, "issue_number"); got != int64(1_000_000_000_000_000) {
|
||||
t.Errorf("tasks[1].issue_number = %#v, want the chat seed", got)
|
||||
}
|
||||
res, err := src.Read(t.Context(), tableNamed(t, StateStoreTables, "resources"))
|
||||
if err != nil {
|
||||
t.Fatalf("Read(resources): %v", err)
|
||||
}
|
||||
if got, ok := res.Value(0, "value").([]byte); !ok || string(got) != "\x00\xff\x10" {
|
||||
t.Errorf("resources[0].value = %#v, want the raw bytes 00 ff 10", res.Value(0, "value"))
|
||||
}
|
||||
}
|
||||
|
||||
func tableNamed(t *testing.T, tables []Table, name string) Table {
|
||||
t.Helper()
|
||||
i := slices.IndexFunc(tables, func(tb Table) bool { return tb.Name == name })
|
||||
if i < 0 {
|
||||
t.Fatalf("no table %q", name)
|
||||
}
|
||||
return tables[i]
|
||||
}
|
||||
|
||||
func TestCompareKeys(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
source []string
|
||||
target []string
|
||||
missing, extra []string
|
||||
}{
|
||||
{name: "equal with the same gap", source: []string{"1", "2", "5"}, target: []string{"5", "1", "2"}},
|
||||
{name: "swallowed row", source: []string{"1", "2", "3"}, target: []string{"1", "3"}, missing: []string{"2"}},
|
||||
{name: "same count, different ids", source: []string{"1", "2"}, target: []string{"1", "9"}, missing: []string{"2"}, extra: []string{"9"}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := CompareKeys(tt.source, tt.target)
|
||||
if !slices.Equal(got.Missing, tt.missing) || !slices.Equal(got.Extra, tt.extra) {
|
||||
t.Errorf("CompareKeys = %+v, want missing %v extra %v", got, tt.missing, tt.extra)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareOrder(t *testing.T) {
|
||||
if got := CompareOrder([]int64{1, 2, 3}, []int64{1, 2, 3}); got != nil {
|
||||
t.Errorf("CompareOrder(equal) = %+v, want nil", got)
|
||||
}
|
||||
got := CompareOrder([]int64{1, 2, 3}, []int64{1, 3, 2})
|
||||
if got == nil || got.Position != 1 || got.Source != 2 || got.Target != 3 {
|
||||
t.Errorf("CompareOrder(swapped) = %+v, want the first divergence at position 1 (2 vs 3)", got)
|
||||
}
|
||||
if got := CompareOrder([]int64{1, 2}, []int64{1}); got == nil || got.Position != 1 {
|
||||
t.Errorf("CompareOrder(short target) = %+v, want a divergence at position 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMicrosecondCollisions constructs D3.8's condition: two events whose
|
||||
// instants differ only below a microsecond, with the later instant carrying
|
||||
// the lower id, so Postgres's (at, id) order would reverse them.
|
||||
func TestMicrosecondCollisions(t *testing.T) {
|
||||
path := fixture(t, "archie.db", stateStoreDDL)
|
||||
exec(t, path, `INSERT INTO events (id, at, kind) VALUES
|
||||
(10, '2026-02-01T00:00:00.000000900Z', 'later instant, lower id'),
|
||||
(11, '2026-02-01T00:00:00.000000100Z', 'earlier instant, higher id'),
|
||||
(12, '2026-02-01T00:00:05.000000100Z', 'same microsecond, ids in order'),
|
||||
(13, '2026-02-01T00:00:05.000000900Z', 'same microsecond, ids in order')`)
|
||||
src := openSource(t, path)
|
||||
got, err := src.MicrosecondCollisions(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("MicrosecondCollisions: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].EarlierID != 11 || got[0].LaterID != 10 {
|
||||
t.Fatalf("MicrosecondCollisions = %+v, want exactly the pair (11 before 10)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrphans(t *testing.T) {
|
||||
path := fixture(t, "archie.db", stateStoreDDL)
|
||||
src := openSource(t, path)
|
||||
for _, fk := range IdentityForeignKeys {
|
||||
got, err := src.Orphans(t.Context(), fk)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("Orphans(%s) on a clean fixture = %v, %v; want none", fk.Table, got, err)
|
||||
}
|
||||
}
|
||||
_ = src.Close()
|
||||
exec(t, path, `INSERT INTO identity_aliases VALUES ('ghost', 'id-missing')`)
|
||||
src = openSource(t, path)
|
||||
got, err := src.Orphans(t.Context(), IdentityForeignKeys[0])
|
||||
if err != nil {
|
||||
t.Fatalf("Orphans: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Key != "ghost" || got[0].Parent != "id-missing" {
|
||||
t.Fatalf("Orphans(identity_aliases) = %+v, want the ghost alias", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDuplicateAliases pins the case-folding hazard: SQLite's NOCASE folds
|
||||
// ASCII only, so two aliases differing in non-ASCII case coexist there and
|
||||
// collide under Postgres's Unicode lower().
|
||||
func TestDuplicateAliases(t *testing.T) {
|
||||
path := fixture(t, "archie.db", stateStoreDDL)
|
||||
exec(t, path, `INSERT INTO identity_aliases VALUES ('ÉMILE', 'id-1'), ('émile', 'id-1')`)
|
||||
src := openSource(t, path)
|
||||
got, err := src.DuplicateAliases(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("DuplicateAliases: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Key != "émile" || len(got[0].Values) != 2 {
|
||||
t.Fatalf("DuplicateAliases = %+v, want one group of two for émile", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuplicateArmedBindings(t *testing.T) {
|
||||
path := fixture(t, "archie.db-eda.sqlite", edaDDL)
|
||||
src := openSource(t, path)
|
||||
if got, err := src.DuplicateBindings(t.Context()); err != nil || len(got) != 0 {
|
||||
t.Fatalf("DuplicateBindings on a clean fixture = %v, %v; want none", got, err)
|
||||
}
|
||||
_ = src.Close()
|
||||
exec(t, path, `INSERT INTO bindings VALUES ('rbind2', '', 'rmap1', 'b2', 'o', 'r', '', 'gh', 'pending_approval', '', 1, 'implement')`)
|
||||
src = openSource(t, path)
|
||||
got, err := src.DuplicateBindings(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("DuplicateBindings: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Key != "gh" || !slices.Equal(got[0].Values, []string{"rbind1", "rbind2"}) {
|
||||
t.Fatalf("DuplicateBindings = %+v, want source gh held by rbind1 and rbind2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSequences(t *testing.T) {
|
||||
src := openSource(t, fixture(t, "archie.db", stateStoreDDL))
|
||||
got, err := src.Sequences(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("Sequences: %v", err)
|
||||
}
|
||||
want := map[string]Sequence{
|
||||
"tasks": {Table: "tasks", MaxID: 4, Next: 5},
|
||||
"transitions": {Table: "transitions", MaxID: 1, Next: 2},
|
||||
"events": {Table: "events", MaxID: 2, Next: 3},
|
||||
"resource_history": {Table: "resource_history", MaxID: 1, Next: 2},
|
||||
"identity_events": {Table: "identity_events", MaxID: 1, Next: 2},
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("Sequences = %+v, want %d tables", got, len(want))
|
||||
}
|
||||
for _, s := range got {
|
||||
if want[s.Table] != s {
|
||||
t.Errorf("Sequences[%s] = %+v, want %+v", s.Table, s, want[s.Table])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSequenceNextSkipsDeletedTop pins that Next comes from AUTOINCREMENT's
|
||||
// high-water mark, not max(id): a deleted top row still reserves its id.
|
||||
func TestSequenceNextSkipsDeletedTop(t *testing.T) {
|
||||
path := fixture(t, "archie.db", stateStoreDDL)
|
||||
exec(t, path, `DELETE FROM tasks WHERE id = 4`)
|
||||
src := openSource(t, path)
|
||||
got, err := src.Sequences(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("Sequences: %v", err)
|
||||
}
|
||||
i := slices.IndexFunc(got, func(s Sequence) bool { return s.Table == "tasks" })
|
||||
if got[i].MaxID != 1 || got[i].Next != 5 {
|
||||
t.Fatalf("Sequences[tasks] = %+v, want max 1, next 5", got[i])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareBlobs(t *testing.T) {
|
||||
src := map[string][]byte{"config": {0, 0xff}, "routing": {1}}
|
||||
if got := CompareBlobs(src, map[string][]byte{"config": {0, 0xff}, "routing": {1}}); len(got) != 0 {
|
||||
t.Errorf("CompareBlobs(equal) = %v, want none", got)
|
||||
}
|
||||
got := CompareBlobs(src, map[string][]byte{"config": {0, 0xfe}})
|
||||
if !slices.Equal(got, []string{"config", "routing"}) {
|
||||
t.Errorf("CompareBlobs = %v, want config (changed) and routing (missing)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexedMessageIDsFindsAStaleIndex(t *testing.T) {
|
||||
path := fixture(t, "archie.db-conversations.sqlite", gatewayDDL)
|
||||
exec(t, path, `DROP TRIGGER messages_ai; INSERT INTO messages (message_id, session_id, text, ts) VALUES ('m3', 's1', 'unindexed', 1002)`)
|
||||
src := openSource(t, path)
|
||||
indexed, err := src.IndexedMessageIDs(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("IndexedMessageIDs: %v", err)
|
||||
}
|
||||
messages, err := src.Read(t.Context(), tableNamed(t, GatewayTables, "messages"))
|
||||
if err != nil {
|
||||
t.Fatalf("Read(messages): %v", err)
|
||||
}
|
||||
got := CompareKeys(messages.Keys(), indexed)
|
||||
if !slices.Equal(got.Missing, []string{"3"}) {
|
||||
t.Fatalf("CompareKeys(messages, index) = %+v, want message 3 missing from the index", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package legacyread
|
||||
|
||||
func idTable(name string, columns ...string) Table {
|
||||
return Table{Name: name, Columns: columns, Key: []string{columns[0]}, OrderBy: columns[0]}
|
||||
}
|
||||
|
||||
// StateStoreTables are the twelve tables of the State Store's archie.db.
|
||||
var StateStoreTables = []Table{
|
||||
idTable("tasks", "id", "owner", "repo", "issue_number", "title", "body", "labels", "status",
|
||||
"workflow", "stage", "branch", "plan", "notes", "pr_number", "tokens_used", "iterations",
|
||||
"attempt", "park_reason", "watch_comment_id", "park_class", "remediation_rounds", "retry_count",
|
||||
"source", "identity", "binding_id", "binding_version", "review_payload",
|
||||
"workflow_definition_version", "workflow_definition_digest", "workflow_definition_yaml",
|
||||
"created_at", "updated_at", "review_cursor"),
|
||||
idTable("transitions", "id", "task_id", "at", "from_status", "to_status", "detail"),
|
||||
{
|
||||
Name: "events",
|
||||
Columns: []string{
|
||||
"id", "at", "kind", "task_id", "repo", "issue", "workflow", "stage", "attempt",
|
||||
"actor_id", "actor_kind", "principal_id", "detail", "data",
|
||||
},
|
||||
Key: []string{"id"},
|
||||
OrderBy: "at, id",
|
||||
},
|
||||
idTable("resources", "kind", "value", "version", "updated_at"),
|
||||
idTable("resource_history", "id", "kind", "value", "version", "actor", "source", "request_id",
|
||||
"expected_version", "current_version", "at"),
|
||||
idTable("channel_status", "id", "name", "state", "detail", "configured", "reload_supported", "observed_at"),
|
||||
{
|
||||
Name: "apply_status",
|
||||
Columns: []string{"process", "kind", "applied_version", "error", "reported_at"},
|
||||
Key: []string{"process", "kind"},
|
||||
OrderBy: "process, kind",
|
||||
},
|
||||
idTable("config_snapshot", "id", "schema", "document", "published_at"),
|
||||
idTable("identities", "id", "kind", "display_name", "lifecycle", "version", "created_at", "updated_at"),
|
||||
idTable("identity_aliases", "alias", "identity_id"),
|
||||
idTable("identity_events", "id", "identity_id", "event_type", "from_lifecycle", "to_lifecycle",
|
||||
"display_name", "actor_id", "source", "request_id", "at"),
|
||||
{
|
||||
Name: "identity_subjects",
|
||||
Columns: []string{"issuer", "subject", "identity_id", "bound_at"},
|
||||
Key: []string{"issuer", "subject"},
|
||||
OrderBy: "issuer, subject",
|
||||
},
|
||||
}
|
||||
|
||||
// EDATables are the six app collections of the PocketBase event store.
|
||||
// PocketBase's own tables (_collections, _params, _superusers, ...) are not
|
||||
// archie data and are not read.
|
||||
var EDATables = []Table{
|
||||
idTable("captures", "id", "authenticated", "body", "content_type", "headers", "received_at",
|
||||
"remote_addr", "source"),
|
||||
idTable("mappings", "id", "created_at", "fields", "name", "source_hint", "updated_at"),
|
||||
idTable("bindings", "id", "created_at", "mapping", "name", "owner", "repo", "secret", "source",
|
||||
"status", "updated_at", "version", "workflow"),
|
||||
idTable("binding_dispatches", "id", "binding", "binding_version", "capture", "dispatched_at", "task_id"),
|
||||
idTable("playbook_dispatches", "id", "action_id", "dispatched_at", "event_id", "playbook_id",
|
||||
"playbook_version"),
|
||||
idTable("tool_calls", "id", "args", "attempt", "called_at", "duration_ms", "error", "result",
|
||||
"task_id", "tool"),
|
||||
}
|
||||
|
||||
// GatewayTables are the Gateway conversation store's three data tables. Its
|
||||
// FTS5 index is read through IndexedMessageIDs.
|
||||
var GatewayTables = []Table{
|
||||
idTable("sessions", "session_id", "platform", "bot_user", "channel_id", "thread_id", "title",
|
||||
"parent_session_id", "branch_name", "created_at", "last_active_at"),
|
||||
idTable("messages", "id", "message_id", "session_id", "source_id", "sender", "text", "ts", "role", "sender_id"),
|
||||
idTable("turns", "turn_id", "session_id", "source_id", "status", "attempt", "owner_id",
|
||||
"input_message_id", "assistant_message_id", "partial_text", "response_text", "tool_calls",
|
||||
"error", "created_at", "updated_at"),
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
package legacyread
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// KeyDiff is the difference between two key sets. A count comparison alone
|
||||
// passes on a swallowed row when ids have gaps; comparing the sets does not.
|
||||
type KeyDiff struct {
|
||||
Missing []string // in the source, not the target
|
||||
Extra []string // in the target, not the source
|
||||
}
|
||||
|
||||
// CompareKeys compares the source's key set with the target's.
|
||||
func CompareKeys(source, target []string) KeyDiff {
|
||||
var d KeyDiff
|
||||
src := setOf(source)
|
||||
tgt := setOf(target)
|
||||
for k := range src {
|
||||
if !tgt[k] {
|
||||
d.Missing = append(d.Missing, k)
|
||||
}
|
||||
}
|
||||
for k := range tgt {
|
||||
if !src[k] {
|
||||
d.Extra = append(d.Extra, k)
|
||||
}
|
||||
}
|
||||
slices.Sort(d.Missing)
|
||||
slices.Sort(d.Extra)
|
||||
return d
|
||||
}
|
||||
|
||||
func setOf(keys []string) map[string]bool {
|
||||
set := make(map[string]bool, len(keys))
|
||||
for _, k := range keys {
|
||||
set[k] = true
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
// Divergence is the first position at which two ordered id sequences differ.
|
||||
// A missing element on one side reads as 0.
|
||||
type Divergence struct {
|
||||
Position int
|
||||
Source, Target int64
|
||||
}
|
||||
|
||||
// CompareOrder compares two id sequences read in cursor order, returning nil
|
||||
// when they are identical.
|
||||
func CompareOrder(source, target []int64) *Divergence {
|
||||
for i := range max(len(source), len(target)) {
|
||||
var s, t int64
|
||||
if i < len(source) {
|
||||
s = source[i]
|
||||
}
|
||||
if i < len(target) {
|
||||
t = target[i]
|
||||
}
|
||||
if s != t || i >= len(source) || i >= len(target) {
|
||||
return &Divergence{Position: i, Source: s, Target: t}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Collision is two events whose instants differ only below a microsecond
|
||||
// while their ids run the other way. Postgres stores microseconds, so its
|
||||
// (at, id) order would put LaterID before EarlierID.
|
||||
type Collision struct {
|
||||
EarlierID, LaterID int64
|
||||
At time.Time // the shared microsecond
|
||||
}
|
||||
|
||||
// eventAtLayouts are the layouts events.at has been written in, newest first.
|
||||
var eventAtLayouts = []string{"2006-01-02T15:04:05.000000000Z", time.RFC3339Nano, time.RFC3339}
|
||||
|
||||
func parseEventAt(s string) (time.Time, error) {
|
||||
for _, layout := range eventAtLayouts {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t.UTC(), nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("unrecognised timestamp %q", s)
|
||||
}
|
||||
|
||||
// MicrosecondCollisions returns every event pair that truncating events.at
|
||||
// to microseconds would reorder.
|
||||
func (s *Source) MicrosecondCollisions(ctx context.Context) ([]Collision, error) {
|
||||
rows, err := s.db.QueryContext(ctx, "SELECT id, at FROM events ORDER BY id")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacyread: events: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
type event struct {
|
||||
id int64
|
||||
at time.Time
|
||||
}
|
||||
byMicro := map[time.Time][]event{}
|
||||
for rows.Next() {
|
||||
var (
|
||||
id int64
|
||||
raw string
|
||||
)
|
||||
if err := rows.Scan(&id, &raw); err != nil {
|
||||
return nil, fmt.Errorf("legacyread: events: %w", err)
|
||||
}
|
||||
at, err := parseEventAt(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacyread: event %d: %w", id, err)
|
||||
}
|
||||
micro := at.Truncate(time.Microsecond)
|
||||
byMicro[micro] = append(byMicro[micro], event{id, at})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("legacyread: events: %w", err)
|
||||
}
|
||||
var out []Collision
|
||||
for micro, events := range byMicro {
|
||||
for i, a := range events {
|
||||
for _, b := range events[i+1:] {
|
||||
// Read in id order, so a.id < b.id.
|
||||
if b.at.Before(a.at) {
|
||||
out = append(out, Collision{EarlierID: b.id, LaterID: a.id, At: micro})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.SortFunc(out, func(x, y Collision) int { return int(x.EarlierID - y.EarlierID) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ForeignKey is a reference SQLite declared but never enforced (the State
|
||||
// Store never set PRAGMA foreign_keys) and Postgres will.
|
||||
type ForeignKey struct {
|
||||
Table, Column string
|
||||
KeyExpr string // SQL naming the child row in a report
|
||||
Parent, ParentC string
|
||||
}
|
||||
|
||||
// IdentityForeignKeys are the three identity references.
|
||||
var IdentityForeignKeys = []ForeignKey{
|
||||
{Table: "identity_aliases", Column: "identity_id", KeyExpr: "alias", Parent: "identities", ParentC: "id"},
|
||||
{Table: "identity_events", Column: "identity_id", KeyExpr: "CAST(id AS TEXT)", Parent: "identities", ParentC: "id"},
|
||||
{Table: "identity_subjects", Column: "identity_id", KeyExpr: "issuer || '/' || subject", Parent: "identities", ParentC: "id"},
|
||||
}
|
||||
|
||||
// Orphan is a child row whose parent does not exist.
|
||||
type Orphan struct {
|
||||
Key, Parent string
|
||||
}
|
||||
|
||||
// Orphans returns fk's child rows with no parent.
|
||||
func (s *Source) Orphans(ctx context.Context, fk ForeignKey) ([]Orphan, error) {
|
||||
query := fmt.Sprintf("SELECT %s, %s FROM %s WHERE %s NOT IN (SELECT %s FROM %s) ORDER BY 1",
|
||||
fk.KeyExpr, fk.Column, fk.Table, fk.Column, fk.ParentC, fk.Parent)
|
||||
rows, err := s.db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacyread: orphans in %s: %w", fk.Table, err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var out []Orphan
|
||||
for rows.Next() {
|
||||
var o Orphan
|
||||
if err := rows.Scan(&o.Key, &o.Parent); err != nil {
|
||||
return nil, fmt.Errorf("legacyread: orphans in %s: %w", fk.Table, err)
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Duplicate is a key the target will hold unique that more than one source
|
||||
// row shares.
|
||||
type Duplicate struct {
|
||||
Key string
|
||||
Values []string
|
||||
}
|
||||
|
||||
// DuplicateAliases groups identity aliases by Unicode lower case. SQLite's
|
||||
// NOCASE folds ASCII only, so aliases that differ in non-ASCII case coexist in
|
||||
// the source and collide on the target's lower(alias) index.
|
||||
func (s *Source) DuplicateAliases(ctx context.Context) ([]Duplicate, error) {
|
||||
aliases, err := s.strings(ctx, "SELECT alias FROM identity_aliases ORDER BY alias")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return duplicates(aliases, aliases, strings.ToLower), nil
|
||||
}
|
||||
|
||||
// DuplicateBindings groups bindings by source. One binding per source is
|
||||
// enforced in Go, not by the legacy schema, so a duplicate is possible and is
|
||||
// refused rather than repaired (D4.2).
|
||||
func (s *Source) DuplicateBindings(ctx context.Context) ([]Duplicate, error) {
|
||||
rows, err := s.Read(ctx, Table{Name: "bindings", Columns: []string{"id", "source"}, OrderBy: "id"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]string, len(rows.Rows))
|
||||
sources := make([]string, len(rows.Rows))
|
||||
for i := range rows.Rows {
|
||||
ids[i] = fmt.Sprint(rows.Value(i, "id"))
|
||||
sources[i] = fmt.Sprint(rows.Value(i, "source"))
|
||||
}
|
||||
return duplicates(sources, ids, func(s string) string { return s }), nil
|
||||
}
|
||||
|
||||
// duplicates groups values by key(keys[i]) and returns the groups of two or
|
||||
// more, sorted by key.
|
||||
func duplicates(keys, values []string, key func(string) string) []Duplicate {
|
||||
groups := map[string][]string{}
|
||||
for i, k := range keys {
|
||||
groups[key(k)] = append(groups[key(k)], values[i])
|
||||
}
|
||||
var out []Duplicate
|
||||
for _, k := range slices.Sorted(maps.Keys(groups)) {
|
||||
if len(groups[k]) > 1 {
|
||||
out = append(out, Duplicate{Key: k, Values: groups[k]})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Sequence is an identity column's state: the highest id present and the id
|
||||
// SQLite would assign next. Next comes from AUTOINCREMENT's high-water mark,
|
||||
// so a deleted top row still reserves its id.
|
||||
type Sequence struct {
|
||||
Table string
|
||||
MaxID, Next int64
|
||||
}
|
||||
|
||||
// sequenceTables are the State Store tables with an AUTOINCREMENT id.
|
||||
var sequenceTables = []string{"tasks", "transitions", "events", "resource_history", "identity_events"}
|
||||
|
||||
// Sequences returns the state of every State Store identity column.
|
||||
func (s *Source) Sequences(ctx context.Context) ([]Sequence, error) {
|
||||
out := make([]Sequence, 0, len(sequenceTables))
|
||||
for _, table := range sequenceTables {
|
||||
seq := Sequence{Table: table}
|
||||
var high int64
|
||||
err := s.db.QueryRowContext(ctx, fmt.Sprintf(
|
||||
"SELECT COALESCE((SELECT max(id) FROM %[1]s), 0), COALESCE((SELECT seq FROM sqlite_sequence WHERE name = '%[1]s'), 0)",
|
||||
table)).Scan(&seq.MaxID, &high)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("legacyread: sequence %s: %w", table, err)
|
||||
}
|
||||
seq.Next = max(seq.MaxID, high) + 1
|
||||
out = append(out, seq)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// CompareBlobs returns, sorted, every key whose bytes differ between source
|
||||
// and target or that only one side holds.
|
||||
func CompareBlobs(source, target map[string][]byte) []string {
|
||||
var out []string
|
||||
for k, v := range source {
|
||||
if t, ok := target[k]; !ok || !bytes.Equal(v, t) {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
for k := range target {
|
||||
if _, ok := source[k]; !ok {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
Reference in new issue
Block a user