feat(edastore): PocketBase-owned collections for event capture and bindings

First slice of the persistence migration, taking the event-capture tables
rather than the task lifecycle. These five tables hold no production rows and
their feature (archie-core-t2db) is unfinished, so they migrate at zero data
risk and the remaining work lands natively instead of being written against
hand-rolled SQL and ported later. They are also the tables that actually want
what PocketBase gives: inspecting a payload nobody has modelled yet, mapping
fields against it, and editing a binding at runtime are the epic's acceptance
criteria, not incidental conveniences.

The task lifecycle deliberately stays on SQLite. tasks/transitions/events
carry FIFO claim ordering, id-cursor pagination and the task-grant map, none
of which gain anything from being hand-editable. binding_dispatches.task_id
is the single seam and stays a plain integer addressing that store.

Idempotency moves from INSERT OR IGNORE on a composite primary key to unique
indexes, so the at-most-once ledgers are enforced by the schema rather than by
each caller remembering the pattern. A save that fails for any other reason is
returned rather than swallowed: treating every error as "already recorded"
would turn a broken ledger into a silent at-least-once dispatcher.

Adds tool_calls, which had no persistence at all. Tool calls were only written
to the on-disk task log (internal/logging), readable as text but never
queryable, filterable or watchable.
This commit is contained in:
sam committed 2026-09-23 01:27:34 +10:00
1 parent ad52528e04
commit 57dbde07da
4 files changed
+642

No files matched your search

@@ -0,0 +1,111 @@
// Package edastore persists the event-capture and playbook-binding tables on
// PocketBase-owned collections.
//
// These are the tables behind the event-capture epic: an inbound webhook is
// captured whether or not anything understands it yet, an operator inspects
// the real payload, maps fields against it, and binds a matcher and mapping
// to a workflow. That loop wants a runtime-editable store with a live view of
// arriving payloads, which is what PocketBase collections give directly.
//
// Unlike the read-only surface in internal/infrastructure/stateadmin, these
// collections are authoritative and writable: the admin UI editing a binding
// IS the feature, not a hazard. The task lifecycle tables stay on SQLite --
// they carry ordering, cursor and task-grant semantics that gain nothing from
// being hand-editable. binding_dispatches.task reaches them by plain id, which
// is the single seam between the two stores.
//
// Idempotency here is schema-enforced. The SQLite implementation relied on
// INSERT OR IGNORE against a composite primary key; a unique index does the
// same job without every caller having to remember the pattern.
package edastore
import (
"errors"
"fmt"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase"
"github.com/pocketbase/pocketbase/core"
_ "modernc.org/sqlite"
)
// Collection names. They are unprefixed because this store owns its database;
// nothing else writes these tables.
const (
CollCaptures = "captures"
CollMappings = "mappings"
CollBindings = "bindings"
CollBindingDispatches = "binding_dispatches"
CollPlaybookDispatches = "playbook_dispatches"
CollToolCalls = "tool_calls"
)
// Binding lifecycle. A binding is only live once a human approves it, which is
// an acceptance criterion of the epic rather than a convention.
const (
StatusDraft = "draft"
StatusApproved = "approved"
)
var (
// ErrBindingNotFound reports an operation naming a binding that is absent.
ErrBindingNotFound = errors.New("edastore: binding not found")
// ErrCaptureNotFound reports a capture that is absent or aged out.
ErrCaptureNotFound = errors.New("edastore: capture not found")
)
// Collections returns every collection this store owns.
func Collections() []string {
return []string{
CollCaptures, CollMappings, CollBindings,
CollBindingDispatches, CollPlaybookDispatches, CollToolCalls,
}
}
// Config describes one store.
type Config struct {
// DBPath is the SQLite file PocketBase owns for these collections.
DBPath string
// DataDir holds PocketBase's auxiliary database and settings.
DataDir string
}
// Store is the PocketBase-backed EDA persistence.
type Store struct{ app *pocketbase.PocketBase }
// Open bootstraps the store and ensures its collections exist.
func Open(cfg Config) (*Store, error) {
if cfg.DBPath == "" || cfg.DataDir == "" {
return nil, errors.New("edastore: DBPath and DataDir are required")
}
app := pocketbase.NewWithConfig(pocketbase.Config{
DefaultDataDir: cfg.DataDir,
HideStartBanner: true,
DBConnect: func(requested string) (*dbx.DB, error) {
if isDataDB(requested) {
requested = cfg.DBPath
}
return dbx.Open("sqlite", requested+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
},
})
if err := app.Bootstrap(); err != nil {
return nil, fmt.Errorf("edastore: bootstrap: %w", err)
}
if err := app.RunAllMigrations(); err != nil {
return nil, fmt.Errorf("edastore: migrations: %w", err)
}
if err := removeDefaultAuthCollection(app); err != nil {
return nil, err
}
if err := ensureCollections(app); err != nil {
return nil, err
}
return &Store{app: app}, nil
}
// App exposes the PocketBase application so one process can serve this store
// and its admin UI from a single bootstrap.
func (s *Store) App() core.App { return s.app }
// Close releases the store's database connections.
func (s *Store) Close() error { return s.app.ClearBootstrap() }
+211
View File
@@ -0,0 +1,211 @@
package edastore
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"github.com/pocketbase/pocketbase/core"
)
// Capture is one inbound event, stored whether or not anything understands it.
type Capture struct {
ID string
Source string
RemoteAddr string
ContentType string
Headers string
Body string
Authenticated bool
}
// Binding ties a matcher and mapping to a workflow.
type Binding struct {
ID string
Name string
Source string
MappingID string
Workflow string
Owner string
Repo string
Version int64
Status string
Secret string
}
// Dispatch is one at-most-once binding dispatch ledger entry.
type Dispatch struct {
BindingID string
BindingVersion int64
CaptureID string
// TaskID addresses the SQLite-owned task tables, which are not migrating.
TaskID int64
}
// PlaybookDispatch is one at-most-once playbook action ledger entry.
type PlaybookDispatch struct {
PlaybookID string
PlaybookVersion string
EventID string
ActionID string
}
// InsertCapture stores one inbound event and returns its id.
func (s *Store) InsertCapture(_ context.Context, c Capture) (string, error) {
collection, err := s.app.FindCollectionByNameOrId(CollCaptures)
if err != nil {
return "", err
}
r := core.NewRecord(collection)
r.Set("source", c.Source)
r.Set("remote_addr", c.RemoteAddr)
r.Set("content_type", c.ContentType)
r.Set("headers", c.Headers)
r.Set("body", c.Body)
r.Set("authenticated", c.Authenticated)
if err := s.app.Save(r); err != nil {
return "", fmt.Errorf("edastore: insert capture: %w", err)
}
return r.Id, nil
}
// Capture returns one stored event verbatim.
func (s *Store) Capture(_ context.Context, id string) (Capture, error) {
r, err := s.app.FindRecordById(CollCaptures, id)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return Capture{}, ErrCaptureNotFound
}
return Capture{}, err
}
return Capture{
ID: r.Id,
Source: r.GetString("source"),
RemoteAddr: r.GetString("remote_addr"),
ContentType: r.GetString("content_type"),
Headers: r.GetString("headers"),
Body: r.GetString("body"),
Authenticated: r.GetBool("authenticated"),
}, nil
}
// InsertBinding stores a new binding. It is always created as a draft: the
// epic requires explicit human approval before a binding can run anything.
func (s *Store) InsertBinding(_ context.Context, b Binding) (string, error) {
collection, err := s.app.FindCollectionByNameOrId(CollBindings)
if err != nil {
return "", err
}
r := core.NewRecord(collection)
r.Set("name", b.Name)
r.Set("source", b.Source)
r.Set("mapping", b.MappingID)
r.Set("workflow", b.Workflow)
r.Set("owner", b.Owner)
r.Set("repo", b.Repo)
r.Set("version", 1)
r.Set("status", StatusDraft)
r.Set("secret", b.Secret)
if err := s.app.Save(r); err != nil {
return "", fmt.Errorf("edastore: insert binding: %w", err)
}
return r.Id, nil
}
// GetBinding returns one binding.
func (s *Store) GetBinding(_ context.Context, id string) (Binding, error) {
r, err := s.app.FindRecordById(CollBindings, id)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return Binding{}, ErrBindingNotFound
}
return Binding{}, err
}
return Binding{
ID: r.Id,
Name: r.GetString("name"),
Source: r.GetString("source"),
MappingID: r.GetString("mapping"),
Workflow: r.GetString("workflow"),
Owner: r.GetString("owner"),
Repo: r.GetString("repo"),
Version: int64(r.GetInt("version")),
Status: r.GetString("status"),
Secret: r.GetString("secret"),
}, nil
}
// ApproveBinding moves a draft binding to approved. It is the only way a
// binding becomes live.
func (s *Store) ApproveBinding(_ context.Context, id string) error {
r, err := s.app.FindRecordById(CollBindings, id)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return ErrBindingNotFound
}
return err
}
r.Set("status", StatusApproved)
r.Set("version", r.GetInt("version")+1)
if err := s.app.Save(r); err != nil {
return fmt.Errorf("edastore: approve binding: %w", err)
}
return nil
}
// RecordDispatch writes one binding dispatch, reporting whether it was new.
// The (binding, capture) uniqueness is enforced by the collection's index, so
// a duplicate is refused by the schema rather than by the caller.
func (s *Store) RecordDispatch(_ context.Context, d Dispatch) (bool, error) {
collection, err := s.app.FindCollectionByNameOrId(CollBindingDispatches)
if err != nil {
return false, err
}
r := core.NewRecord(collection)
r.Set("binding", d.BindingID)
r.Set("binding_version", d.BindingVersion)
r.Set("capture", d.CaptureID)
r.Set("task_id", d.TaskID)
return savedOrDuplicate(s.app, r, "edastore: record dispatch")
}
// RecordPlaybookDispatch writes one playbook action dispatch, reporting
// whether it was new.
func (s *Store) RecordPlaybookDispatch(_ context.Context, d PlaybookDispatch) (bool, error) {
collection, err := s.app.FindCollectionByNameOrId(CollPlaybookDispatches)
if err != nil {
return false, err
}
r := core.NewRecord(collection)
r.Set("playbook_id", d.PlaybookID)
r.Set("playbook_version", d.PlaybookVersion)
r.Set("event_id", d.EventID)
r.Set("action_id", d.ActionID)
return savedOrDuplicate(s.app, r, "edastore: record playbook dispatch")
}
// savedOrDuplicate saves a ledger record, translating the unique-index
// violation into "already recorded" rather than an error. Anything else is a
// real failure and is returned: silently swallowing every save error here
// would turn a broken ledger into a silent at-least-once dispatcher.
func savedOrDuplicate(app core.App, r *core.Record, what string) (bool, error) {
err := app.Save(r)
switch {
case err == nil:
return true, nil
case isUniqueViolation(err):
return false, nil
default:
return false, fmt.Errorf("%s: %w", what, err)
}
}
// isUniqueViolation reports whether err is the unique-index refusal. PocketBase
// surfaces it as a validation failure rather than a typed error, so the text is
// what there is to match on.
func isUniqueViolation(err error) bool {
text := strings.ToLower(err.Error())
return strings.Contains(text, "unique") || strings.Contains(text, "already exists")
}
+191
View File
@@ -0,0 +1,191 @@
package edastore
import (
"database/sql"
"errors"
"fmt"
"path/filepath"
"github.com/pocketbase/pocketbase/core"
)
// isDataDB reports whether PocketBase is asking for its main database, as
// opposed to the auxiliary one it keeps for logs and settings.
func isDataDB(path string) bool { return filepath.Base(path) == "data.db" }
// removeDefaultAuthCollection deletes the open-signup "users" collection
// PocketBase creates on first bootstrap. This store has no end users, and its
// createRule is open, so leaving it would accept unauthenticated writes.
func removeDefaultAuthCollection(app core.App) error {
c, err := app.FindCollectionByNameOrId("users")
switch {
case errors.Is(err, sql.ErrNoRows):
return nil
case err != nil:
return fmt.Errorf("edastore: look up default users collection: %w", err)
}
if err := app.Delete(c); err != nil {
return fmt.Errorf("edastore: remove default users collection: %w", err)
}
return nil
}
// ensureCollections creates each collection once and is safe to re-run: the
// store bootstraps on every process start.
func ensureCollections(app core.App) error {
for _, build := range []func(core.App) error{
ensureCaptures, ensureMappings, ensureBindings,
ensureBindingDispatches, ensurePlaybookDispatches, ensureToolCalls,
} {
if err := build(app); err != nil {
return err
}
}
return nil
}
// collection returns the named collection, or a new base collection when it
// does not exist yet. A lookup that failed for any other reason is an error:
// creating a second collection over a live one on a transient failure would be
// a schema change made for the wrong reason.
func collection(app core.App, name string) (*core.Collection, bool, error) {
c, err := app.FindCollectionByNameOrId(name)
switch {
case errors.Is(err, sql.ErrNoRows):
return core.NewBaseCollection(name), true, nil
case err != nil:
return nil, false, fmt.Errorf("edastore: look up collection %q: %w", name, err)
}
return c, false, nil
}
func ensureCaptures(app core.App) error {
c, fresh, err := collection(app, CollCaptures)
if err != nil || !fresh {
return err
}
c.Fields.Add(
&core.TextField{Name: "source", Required: true},
&core.TextField{Name: "remote_addr"},
&core.TextField{Name: "content_type"},
// Headers and body are stored verbatim. Field mapping is designed
// against the real payload, so nothing here may be normalised away.
&core.JSONField{Name: "headers", MaxSize: 1 << 20},
&core.TextField{Name: "body", Max: 1 << 20},
&core.BoolField{Name: "authenticated"},
&core.AutodateField{Name: "received_at", OnCreate: true},
)
c.AddIndex("idx_captures_source", false, "source, received_at", "")
return save(app, c)
}
func ensureMappings(app core.App) error {
c, fresh, err := collection(app, CollMappings)
if err != nil || !fresh {
return err
}
c.Fields.Add(
&core.TextField{Name: "name", Required: true},
&core.TextField{Name: "source_hint"},
&core.JSONField{Name: "fields", MaxSize: 1 << 18},
&core.AutodateField{Name: "created_at", OnCreate: true},
&core.AutodateField{Name: "updated_at", OnCreate: true, OnUpdate: true},
)
return save(app, c)
}
func ensureBindings(app core.App) error {
c, fresh, err := collection(app, CollBindings)
if err != nil || !fresh {
return err
}
mappings, err := app.FindCollectionByNameOrId(CollMappings)
if err != nil {
return fmt.Errorf("edastore: bindings need mappings: %w", err)
}
c.Fields.Add(
&core.TextField{Name: "name", Required: true},
&core.TextField{Name: "source", Required: true},
&core.RelationField{Name: "mapping", CollectionId: mappings.Id, MaxSelect: 1},
&core.TextField{Name: "workflow"},
&core.TextField{Name: "owner"},
&core.TextField{Name: "repo"},
&core.NumberField{Name: "version"},
// status carries the approval gate. It is deliberately not defaulted
// to anything live: a binding reaches "approved" only by ApproveBinding.
&core.SelectField{Name: "status", Values: []string{StatusDraft, StatusApproved}, MaxSelect: 1},
&core.TextField{Name: "secret"},
&core.AutodateField{Name: "created_at", OnCreate: true},
&core.AutodateField{Name: "updated_at", OnCreate: true, OnUpdate: true},
)
c.AddIndex("idx_bindings_source_status", false, "source, status", "")
return save(app, c)
}
func ensureBindingDispatches(app core.App) error {
c, fresh, err := collection(app, CollBindingDispatches)
if err != nil || !fresh {
return err
}
c.Fields.Add(
&core.TextField{Name: "binding", Required: true},
&core.NumberField{Name: "binding_version"},
&core.TextField{Name: "capture", Required: true},
// task_id is the seam to the SQLite-owned task tables. It stays a
// plain integer id rather than a relation precisely because the task
// lifecycle is not migrating.
&core.NumberField{Name: "task_id"},
&core.AutodateField{Name: "dispatched_at", OnCreate: true},
)
// At-most-once, enforced by the schema rather than by every caller
// remembering INSERT OR IGNORE.
c.AddIndex("idx_binding_dispatch_once", true, "binding, capture", "")
return save(app, c)
}
func ensurePlaybookDispatches(app core.App) error {
c, fresh, err := collection(app, CollPlaybookDispatches)
if err != nil || !fresh {
return err
}
c.Fields.Add(
&core.TextField{Name: "playbook_id", Required: true},
&core.TextField{Name: "playbook_version", Required: true},
&core.TextField{Name: "event_id", Required: true},
&core.TextField{Name: "action_id", Required: true},
&core.AutodateField{Name: "dispatched_at", OnCreate: true},
)
c.AddIndex("idx_playbook_dispatch_once", true,
"playbook_id, playbook_version, event_id, action_id", "")
return save(app, c)
}
// ensureToolCalls adds persistence that did not exist before. Tool calls were
// only ever written to the on-disk task log (internal/logging), so they could
// be read back as text but never queried, filtered or watched. As a collection
// they become all three.
func ensureToolCalls(app core.App) error {
c, fresh, err := collection(app, CollToolCalls)
if err != nil || !fresh {
return err
}
c.Fields.Add(
&core.NumberField{Name: "task_id"},
&core.NumberField{Name: "attempt"},
&core.TextField{Name: "tool", Required: true},
&core.JSONField{Name: "args", MaxSize: 1 << 18},
&core.JSONField{Name: "result", MaxSize: 1 << 18},
&core.TextField{Name: "error"},
&core.NumberField{Name: "duration_ms"},
&core.AutodateField{Name: "called_at", OnCreate: true},
)
c.AddIndex("idx_tool_calls_task", false, "task_id, attempt", "")
return save(app, c)
}
func save(app core.App, c *core.Collection) error {
if err := app.Save(c); err != nil {
return fmt.Errorf("edastore: create collection %q: %w", c.Name, err)
}
return nil
}
@@ -0,0 +1,129 @@
package edastore_test
import (
"errors"
"path/filepath"
"testing"
"github.com/pocketbase/pocketbase/core"
"github.com/samcharles93/archie-core/internal/infrastructure/edastore"
)
func newStore(t *testing.T) *edastore.Store {
t.Helper()
dir := t.TempDir()
st, err := edastore.Open(edastore.Config{
DBPath: filepath.Join(dir, "eda.sqlite"),
DataDir: filepath.Join(dir, "pb_data"),
})
if err != nil {
t.Fatalf("edastore.Open() error = %v", err)
}
t.Cleanup(func() { _ = st.Close() })
return st
}
// TestCollectionsAreCreated pins that every EDA collection exists and is a
// base collection, not a view: unlike the read-only operator surface, these
// are the authoritative tables and the admin UI is expected to edit them.
func TestCollectionsAreCreated(t *testing.T) {
st := newStore(t)
for _, name := range edastore.Collections() {
c, err := st.App().FindCollectionByNameOrId(name)
if err != nil {
t.Fatalf("collection %q missing: %v", name, err)
}
if c.IsView() {
t.Errorf("collection %q is a view; EDA tables are authoritative and must be writable", name)
}
}
}
// TestBindingDispatchIsIdempotent pins the at-most-once ledger. The SQLite
// implementation got this from INSERT OR IGNORE on a composite primary key
// (bindings.go:365); here it must come from a unique index, so a duplicate is
// refused by the schema rather than by the caller remembering to ignore it.
func TestBindingDispatchIsIdempotent(t *testing.T) {
st := newStore(t)
first, err := st.RecordDispatch(t.Context(), edastore.Dispatch{
BindingID: "b1", BindingVersion: 1, CaptureID: "c1", TaskID: 42,
})
if err != nil {
t.Fatalf("first RecordDispatch() error = %v", err)
}
if !first {
t.Fatal("first RecordDispatch() = false, want true: a fresh dispatch must be recorded")
}
second, err := st.RecordDispatch(t.Context(), edastore.Dispatch{
BindingID: "b1", BindingVersion: 1, CaptureID: "c1", TaskID: 99,
})
if err != nil {
t.Fatalf("second RecordDispatch() error = %v", err)
}
if second {
t.Error("second RecordDispatch() = true, want false: (binding, capture) is an at-most-once ledger")
}
}
// TestPlaybookDispatchIsIdempotent pins the same at-most-once rule on the
// four-part playbook key (playbook, version, event, action).
func TestPlaybookDispatchIsIdempotent(t *testing.T) {
st := newStore(t)
key := edastore.PlaybookDispatch{PlaybookID: "p1", PlaybookVersion: "v1", EventID: "e1", ActionID: "a1"}
if ok, err := st.RecordPlaybookDispatch(t.Context(), key); err != nil || !ok {
t.Fatalf("first RecordPlaybookDispatch() = %v, %v; want true, nil", ok, err)
}
if ok, err := st.RecordPlaybookDispatch(t.Context(), key); err != nil || ok {
t.Fatalf("second RecordPlaybookDispatch() = %v, %v; want false, nil", ok, err)
}
}
// TestBindingStartsAsDraft pins the approval gate from the epic's acceptance
// criteria: "A binding cannot go live without explicit human approval." A
// binding created without a status must not default to anything live.
func TestBindingStartsAsDraft(t *testing.T) {
st := newStore(t)
id, err := st.InsertBinding(t.Context(), edastore.Binding{Name: "n", Source: "sentry", Workflow: "implement"})
if err != nil {
t.Fatalf("InsertBinding() error = %v", err)
}
b, err := st.GetBinding(t.Context(), id)
if err != nil {
t.Fatalf("GetBinding() error = %v", err)
}
if b.Status != edastore.StatusDraft {
t.Errorf("new binding status = %q, want %q: a binding must not be live before approval", b.Status, edastore.StatusDraft)
}
}
// TestApproveRejectsUnknownBinding pins the sentinel the gRPC layer maps.
func TestApproveRejectsUnknownBinding(t *testing.T) {
st := newStore(t)
if err := st.ApproveBinding(t.Context(), "nosuchbinding"); !errors.Is(err, edastore.ErrBindingNotFound) {
t.Errorf("ApproveBinding(unknown) error = %v, want ErrBindingNotFound", err)
}
}
// TestCaptureRoundTripsPayload pins the epic's core requirement: an event
// arrives with a payload nobody has modelled yet and must be readable back
// verbatim so an operator can map fields against it.
func TestCaptureRoundTripsPayload(t *testing.T) {
st := newStore(t)
body := `{"action":"created","issue":{"number":7,"title":"it broke"}}`
id, err := st.InsertCapture(t.Context(), edastore.Capture{
Source: "sentry", Body: body, Headers: `{"X-Hook":"1"}`, ContentType: "application/json",
})
if err != nil {
t.Fatalf("InsertCapture() error = %v", err)
}
got, err := st.Capture(t.Context(), id)
if err != nil {
t.Fatalf("Capture() error = %v", err)
}
if got.Body != body {
t.Errorf("Body round-trip = %q, want %q: the raw payload is what field mapping is designed against", got.Body, body)
}
}
var _ = core.Collection{}