diff --git a/internal/infrastructure/edastore/edastore.go b/internal/infrastructure/edastore/edastore.go new file mode 100644 index 000000000..2adca1db6 --- /dev/null +++ b/internal/infrastructure/edastore/edastore.go @@ -0,0 +1,111 @@ +// Package edastore persists the event-capture and playbook-binding tables on +// PocketBase-owned collections. +// +// These are the tables behind the event-capture epic: an inbound webhook is +// captured whether or not anything understands it yet, an operator inspects +// the real payload, maps fields against it, and binds a matcher and mapping +// to a workflow. That loop wants a runtime-editable store with a live view of +// arriving payloads, which is what PocketBase collections give directly. +// +// Unlike the read-only surface in internal/infrastructure/stateadmin, these +// collections are authoritative and writable: the admin UI editing a binding +// IS the feature, not a hazard. The task lifecycle tables stay on SQLite -- +// they carry ordering, cursor and task-grant semantics that gain nothing from +// being hand-editable. binding_dispatches.task reaches them by plain id, which +// is the single seam between the two stores. +// +// Idempotency here is schema-enforced. The SQLite implementation relied on +// INSERT OR IGNORE against a composite primary key; a unique index does the +// same job without every caller having to remember the pattern. +package edastore + +import ( + "errors" + "fmt" + + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/core" + _ "modernc.org/sqlite" +) + +// Collection names. They are unprefixed because this store owns its database; +// nothing else writes these tables. +const ( + CollCaptures = "captures" + CollMappings = "mappings" + CollBindings = "bindings" + CollBindingDispatches = "binding_dispatches" + CollPlaybookDispatches = "playbook_dispatches" + CollToolCalls = "tool_calls" +) + +// Binding lifecycle. A binding is only live once a human approves it, which is +// an acceptance criterion of the epic rather than a convention. +const ( + StatusDraft = "draft" + StatusApproved = "approved" +) + +var ( + // ErrBindingNotFound reports an operation naming a binding that is absent. + ErrBindingNotFound = errors.New("edastore: binding not found") + // ErrCaptureNotFound reports a capture that is absent or aged out. + ErrCaptureNotFound = errors.New("edastore: capture not found") +) + +// Collections returns every collection this store owns. +func Collections() []string { + return []string{ + CollCaptures, CollMappings, CollBindings, + CollBindingDispatches, CollPlaybookDispatches, CollToolCalls, + } +} + +// Config describes one store. +type Config struct { + // DBPath is the SQLite file PocketBase owns for these collections. + DBPath string + // DataDir holds PocketBase's auxiliary database and settings. + DataDir string +} + +// Store is the PocketBase-backed EDA persistence. +type Store struct{ app *pocketbase.PocketBase } + +// Open bootstraps the store and ensures its collections exist. +func Open(cfg Config) (*Store, error) { + if cfg.DBPath == "" || cfg.DataDir == "" { + return nil, errors.New("edastore: DBPath and DataDir are required") + } + app := pocketbase.NewWithConfig(pocketbase.Config{ + DefaultDataDir: cfg.DataDir, + HideStartBanner: true, + DBConnect: func(requested string) (*dbx.DB, error) { + if isDataDB(requested) { + requested = cfg.DBPath + } + return dbx.Open("sqlite", requested+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + }, + }) + if err := app.Bootstrap(); err != nil { + return nil, fmt.Errorf("edastore: bootstrap: %w", err) + } + if err := app.RunAllMigrations(); err != nil { + return nil, fmt.Errorf("edastore: migrations: %w", err) + } + if err := removeDefaultAuthCollection(app); err != nil { + return nil, err + } + if err := ensureCollections(app); err != nil { + return nil, err + } + return &Store{app: app}, nil +} + +// App exposes the PocketBase application so one process can serve this store +// and its admin UI from a single bootstrap. +func (s *Store) App() core.App { return s.app } + +// Close releases the store's database connections. +func (s *Store) Close() error { return s.app.ClearBootstrap() } diff --git a/internal/infrastructure/edastore/records.go b/internal/infrastructure/edastore/records.go new file mode 100644 index 000000000..f079c7830 --- /dev/null +++ b/internal/infrastructure/edastore/records.go @@ -0,0 +1,211 @@ +package edastore + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + + "github.com/pocketbase/pocketbase/core" +) + +// Capture is one inbound event, stored whether or not anything understands it. +type Capture struct { + ID string + Source string + RemoteAddr string + ContentType string + Headers string + Body string + Authenticated bool +} + +// Binding ties a matcher and mapping to a workflow. +type Binding struct { + ID string + Name string + Source string + MappingID string + Workflow string + Owner string + Repo string + Version int64 + Status string + Secret string +} + +// Dispatch is one at-most-once binding dispatch ledger entry. +type Dispatch struct { + BindingID string + BindingVersion int64 + CaptureID string + // TaskID addresses the SQLite-owned task tables, which are not migrating. + TaskID int64 +} + +// PlaybookDispatch is one at-most-once playbook action ledger entry. +type PlaybookDispatch struct { + PlaybookID string + PlaybookVersion string + EventID string + ActionID string +} + +// InsertCapture stores one inbound event and returns its id. +func (s *Store) InsertCapture(_ context.Context, c Capture) (string, error) { + collection, err := s.app.FindCollectionByNameOrId(CollCaptures) + if err != nil { + return "", err + } + r := core.NewRecord(collection) + r.Set("source", c.Source) + r.Set("remote_addr", c.RemoteAddr) + r.Set("content_type", c.ContentType) + r.Set("headers", c.Headers) + r.Set("body", c.Body) + r.Set("authenticated", c.Authenticated) + if err := s.app.Save(r); err != nil { + return "", fmt.Errorf("edastore: insert capture: %w", err) + } + return r.Id, nil +} + +// Capture returns one stored event verbatim. +func (s *Store) Capture(_ context.Context, id string) (Capture, error) { + r, err := s.app.FindRecordById(CollCaptures, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Capture{}, ErrCaptureNotFound + } + return Capture{}, err + } + return Capture{ + ID: r.Id, + Source: r.GetString("source"), + RemoteAddr: r.GetString("remote_addr"), + ContentType: r.GetString("content_type"), + Headers: r.GetString("headers"), + Body: r.GetString("body"), + Authenticated: r.GetBool("authenticated"), + }, nil +} + +// InsertBinding stores a new binding. It is always created as a draft: the +// epic requires explicit human approval before a binding can run anything. +func (s *Store) InsertBinding(_ context.Context, b Binding) (string, error) { + collection, err := s.app.FindCollectionByNameOrId(CollBindings) + if err != nil { + return "", err + } + r := core.NewRecord(collection) + r.Set("name", b.Name) + r.Set("source", b.Source) + r.Set("mapping", b.MappingID) + r.Set("workflow", b.Workflow) + r.Set("owner", b.Owner) + r.Set("repo", b.Repo) + r.Set("version", 1) + r.Set("status", StatusDraft) + r.Set("secret", b.Secret) + if err := s.app.Save(r); err != nil { + return "", fmt.Errorf("edastore: insert binding: %w", err) + } + return r.Id, nil +} + +// GetBinding returns one binding. +func (s *Store) GetBinding(_ context.Context, id string) (Binding, error) { + r, err := s.app.FindRecordById(CollBindings, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Binding{}, ErrBindingNotFound + } + return Binding{}, err + } + return Binding{ + ID: r.Id, + Name: r.GetString("name"), + Source: r.GetString("source"), + MappingID: r.GetString("mapping"), + Workflow: r.GetString("workflow"), + Owner: r.GetString("owner"), + Repo: r.GetString("repo"), + Version: int64(r.GetInt("version")), + Status: r.GetString("status"), + Secret: r.GetString("secret"), + }, nil +} + +// ApproveBinding moves a draft binding to approved. It is the only way a +// binding becomes live. +func (s *Store) ApproveBinding(_ context.Context, id string) error { + r, err := s.app.FindRecordById(CollBindings, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return ErrBindingNotFound + } + return err + } + r.Set("status", StatusApproved) + r.Set("version", r.GetInt("version")+1) + if err := s.app.Save(r); err != nil { + return fmt.Errorf("edastore: approve binding: %w", err) + } + return nil +} + +// RecordDispatch writes one binding dispatch, reporting whether it was new. +// The (binding, capture) uniqueness is enforced by the collection's index, so +// a duplicate is refused by the schema rather than by the caller. +func (s *Store) RecordDispatch(_ context.Context, d Dispatch) (bool, error) { + collection, err := s.app.FindCollectionByNameOrId(CollBindingDispatches) + if err != nil { + return false, err + } + r := core.NewRecord(collection) + r.Set("binding", d.BindingID) + r.Set("binding_version", d.BindingVersion) + r.Set("capture", d.CaptureID) + r.Set("task_id", d.TaskID) + return savedOrDuplicate(s.app, r, "edastore: record dispatch") +} + +// RecordPlaybookDispatch writes one playbook action dispatch, reporting +// whether it was new. +func (s *Store) RecordPlaybookDispatch(_ context.Context, d PlaybookDispatch) (bool, error) { + collection, err := s.app.FindCollectionByNameOrId(CollPlaybookDispatches) + if err != nil { + return false, err + } + r := core.NewRecord(collection) + r.Set("playbook_id", d.PlaybookID) + r.Set("playbook_version", d.PlaybookVersion) + r.Set("event_id", d.EventID) + r.Set("action_id", d.ActionID) + return savedOrDuplicate(s.app, r, "edastore: record playbook dispatch") +} + +// savedOrDuplicate saves a ledger record, translating the unique-index +// violation into "already recorded" rather than an error. Anything else is a +// real failure and is returned: silently swallowing every save error here +// would turn a broken ledger into a silent at-least-once dispatcher. +func savedOrDuplicate(app core.App, r *core.Record, what string) (bool, error) { + err := app.Save(r) + switch { + case err == nil: + return true, nil + case isUniqueViolation(err): + return false, nil + default: + return false, fmt.Errorf("%s: %w", what, err) + } +} + +// isUniqueViolation reports whether err is the unique-index refusal. PocketBase +// surfaces it as a validation failure rather than a typed error, so the text is +// what there is to match on. +func isUniqueViolation(err error) bool { + text := strings.ToLower(err.Error()) + return strings.Contains(text, "unique") || strings.Contains(text, "already exists") +} diff --git a/internal/infrastructure/edastore/schema.go b/internal/infrastructure/edastore/schema.go new file mode 100644 index 000000000..f7c83f0ad --- /dev/null +++ b/internal/infrastructure/edastore/schema.go @@ -0,0 +1,191 @@ +package edastore + +import ( + "database/sql" + "errors" + "fmt" + "path/filepath" + + "github.com/pocketbase/pocketbase/core" +) + +// isDataDB reports whether PocketBase is asking for its main database, as +// opposed to the auxiliary one it keeps for logs and settings. +func isDataDB(path string) bool { return filepath.Base(path) == "data.db" } + +// removeDefaultAuthCollection deletes the open-signup "users" collection +// PocketBase creates on first bootstrap. This store has no end users, and its +// createRule is open, so leaving it would accept unauthenticated writes. +func removeDefaultAuthCollection(app core.App) error { + c, err := app.FindCollectionByNameOrId("users") + switch { + case errors.Is(err, sql.ErrNoRows): + return nil + case err != nil: + return fmt.Errorf("edastore: look up default users collection: %w", err) + } + if err := app.Delete(c); err != nil { + return fmt.Errorf("edastore: remove default users collection: %w", err) + } + return nil +} + +// ensureCollections creates each collection once and is safe to re-run: the +// store bootstraps on every process start. +func ensureCollections(app core.App) error { + for _, build := range []func(core.App) error{ + ensureCaptures, ensureMappings, ensureBindings, + ensureBindingDispatches, ensurePlaybookDispatches, ensureToolCalls, + } { + if err := build(app); err != nil { + return err + } + } + return nil +} + +// collection returns the named collection, or a new base collection when it +// does not exist yet. A lookup that failed for any other reason is an error: +// creating a second collection over a live one on a transient failure would be +// a schema change made for the wrong reason. +func collection(app core.App, name string) (*core.Collection, bool, error) { + c, err := app.FindCollectionByNameOrId(name) + switch { + case errors.Is(err, sql.ErrNoRows): + return core.NewBaseCollection(name), true, nil + case err != nil: + return nil, false, fmt.Errorf("edastore: look up collection %q: %w", name, err) + } + return c, false, nil +} + +func ensureCaptures(app core.App) error { + c, fresh, err := collection(app, CollCaptures) + if err != nil || !fresh { + return err + } + c.Fields.Add( + &core.TextField{Name: "source", Required: true}, + &core.TextField{Name: "remote_addr"}, + &core.TextField{Name: "content_type"}, + // Headers and body are stored verbatim. Field mapping is designed + // against the real payload, so nothing here may be normalised away. + &core.JSONField{Name: "headers", MaxSize: 1 << 20}, + &core.TextField{Name: "body", Max: 1 << 20}, + &core.BoolField{Name: "authenticated"}, + &core.AutodateField{Name: "received_at", OnCreate: true}, + ) + c.AddIndex("idx_captures_source", false, "source, received_at", "") + return save(app, c) +} + +func ensureMappings(app core.App) error { + c, fresh, err := collection(app, CollMappings) + if err != nil || !fresh { + return err + } + c.Fields.Add( + &core.TextField{Name: "name", Required: true}, + &core.TextField{Name: "source_hint"}, + &core.JSONField{Name: "fields", MaxSize: 1 << 18}, + &core.AutodateField{Name: "created_at", OnCreate: true}, + &core.AutodateField{Name: "updated_at", OnCreate: true, OnUpdate: true}, + ) + return save(app, c) +} + +func ensureBindings(app core.App) error { + c, fresh, err := collection(app, CollBindings) + if err != nil || !fresh { + return err + } + mappings, err := app.FindCollectionByNameOrId(CollMappings) + if err != nil { + return fmt.Errorf("edastore: bindings need mappings: %w", err) + } + c.Fields.Add( + &core.TextField{Name: "name", Required: true}, + &core.TextField{Name: "source", Required: true}, + &core.RelationField{Name: "mapping", CollectionId: mappings.Id, MaxSelect: 1}, + &core.TextField{Name: "workflow"}, + &core.TextField{Name: "owner"}, + &core.TextField{Name: "repo"}, + &core.NumberField{Name: "version"}, + // status carries the approval gate. It is deliberately not defaulted + // to anything live: a binding reaches "approved" only by ApproveBinding. + &core.SelectField{Name: "status", Values: []string{StatusDraft, StatusApproved}, MaxSelect: 1}, + &core.TextField{Name: "secret"}, + &core.AutodateField{Name: "created_at", OnCreate: true}, + &core.AutodateField{Name: "updated_at", OnCreate: true, OnUpdate: true}, + ) + c.AddIndex("idx_bindings_source_status", false, "source, status", "") + return save(app, c) +} + +func ensureBindingDispatches(app core.App) error { + c, fresh, err := collection(app, CollBindingDispatches) + if err != nil || !fresh { + return err + } + c.Fields.Add( + &core.TextField{Name: "binding", Required: true}, + &core.NumberField{Name: "binding_version"}, + &core.TextField{Name: "capture", Required: true}, + // task_id is the seam to the SQLite-owned task tables. It stays a + // plain integer id rather than a relation precisely because the task + // lifecycle is not migrating. + &core.NumberField{Name: "task_id"}, + &core.AutodateField{Name: "dispatched_at", OnCreate: true}, + ) + // At-most-once, enforced by the schema rather than by every caller + // remembering INSERT OR IGNORE. + c.AddIndex("idx_binding_dispatch_once", true, "binding, capture", "") + return save(app, c) +} + +func ensurePlaybookDispatches(app core.App) error { + c, fresh, err := collection(app, CollPlaybookDispatches) + if err != nil || !fresh { + return err + } + c.Fields.Add( + &core.TextField{Name: "playbook_id", Required: true}, + &core.TextField{Name: "playbook_version", Required: true}, + &core.TextField{Name: "event_id", Required: true}, + &core.TextField{Name: "action_id", Required: true}, + &core.AutodateField{Name: "dispatched_at", OnCreate: true}, + ) + c.AddIndex("idx_playbook_dispatch_once", true, + "playbook_id, playbook_version, event_id, action_id", "") + return save(app, c) +} + +// ensureToolCalls adds persistence that did not exist before. Tool calls were +// only ever written to the on-disk task log (internal/logging), so they could +// be read back as text but never queried, filtered or watched. As a collection +// they become all three. +func ensureToolCalls(app core.App) error { + c, fresh, err := collection(app, CollToolCalls) + if err != nil || !fresh { + return err + } + c.Fields.Add( + &core.NumberField{Name: "task_id"}, + &core.NumberField{Name: "attempt"}, + &core.TextField{Name: "tool", Required: true}, + &core.JSONField{Name: "args", MaxSize: 1 << 18}, + &core.JSONField{Name: "result", MaxSize: 1 << 18}, + &core.TextField{Name: "error"}, + &core.NumberField{Name: "duration_ms"}, + &core.AutodateField{Name: "called_at", OnCreate: true}, + ) + c.AddIndex("idx_tool_calls_task", false, "task_id, attempt", "") + return save(app, c) +} + +func save(app core.App, c *core.Collection) error { + if err := app.Save(c); err != nil { + return fmt.Errorf("edastore: create collection %q: %w", c.Name, err) + } + return nil +} diff --git a/internal/infrastructure/edastore/schema_test.go b/internal/infrastructure/edastore/schema_test.go new file mode 100644 index 000000000..242d64534 --- /dev/null +++ b/internal/infrastructure/edastore/schema_test.go @@ -0,0 +1,129 @@ +package edastore_test + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/pocketbase/pocketbase/core" + + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" +) + +func newStore(t *testing.T) *edastore.Store { + t.Helper() + dir := t.TempDir() + st, err := edastore.Open(edastore.Config{ + DBPath: filepath.Join(dir, "eda.sqlite"), + DataDir: filepath.Join(dir, "pb_data"), + }) + if err != nil { + t.Fatalf("edastore.Open() error = %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +} + +// TestCollectionsAreCreated pins that every EDA collection exists and is a +// base collection, not a view: unlike the read-only operator surface, these +// are the authoritative tables and the admin UI is expected to edit them. +func TestCollectionsAreCreated(t *testing.T) { + st := newStore(t) + for _, name := range edastore.Collections() { + c, err := st.App().FindCollectionByNameOrId(name) + if err != nil { + t.Fatalf("collection %q missing: %v", name, err) + } + if c.IsView() { + t.Errorf("collection %q is a view; EDA tables are authoritative and must be writable", name) + } + } +} + +// TestBindingDispatchIsIdempotent pins the at-most-once ledger. The SQLite +// implementation got this from INSERT OR IGNORE on a composite primary key +// (bindings.go:365); here it must come from a unique index, so a duplicate is +// refused by the schema rather than by the caller remembering to ignore it. +func TestBindingDispatchIsIdempotent(t *testing.T) { + st := newStore(t) + first, err := st.RecordDispatch(t.Context(), edastore.Dispatch{ + BindingID: "b1", BindingVersion: 1, CaptureID: "c1", TaskID: 42, + }) + if err != nil { + t.Fatalf("first RecordDispatch() error = %v", err) + } + if !first { + t.Fatal("first RecordDispatch() = false, want true: a fresh dispatch must be recorded") + } + second, err := st.RecordDispatch(t.Context(), edastore.Dispatch{ + BindingID: "b1", BindingVersion: 1, CaptureID: "c1", TaskID: 99, + }) + if err != nil { + t.Fatalf("second RecordDispatch() error = %v", err) + } + if second { + t.Error("second RecordDispatch() = true, want false: (binding, capture) is an at-most-once ledger") + } +} + +// TestPlaybookDispatchIsIdempotent pins the same at-most-once rule on the +// four-part playbook key (playbook, version, event, action). +func TestPlaybookDispatchIsIdempotent(t *testing.T) { + st := newStore(t) + key := edastore.PlaybookDispatch{PlaybookID: "p1", PlaybookVersion: "v1", EventID: "e1", ActionID: "a1"} + if ok, err := st.RecordPlaybookDispatch(t.Context(), key); err != nil || !ok { + t.Fatalf("first RecordPlaybookDispatch() = %v, %v; want true, nil", ok, err) + } + if ok, err := st.RecordPlaybookDispatch(t.Context(), key); err != nil || ok { + t.Fatalf("second RecordPlaybookDispatch() = %v, %v; want false, nil", ok, err) + } +} + +// TestBindingStartsAsDraft pins the approval gate from the epic's acceptance +// criteria: "A binding cannot go live without explicit human approval." A +// binding created without a status must not default to anything live. +func TestBindingStartsAsDraft(t *testing.T) { + st := newStore(t) + id, err := st.InsertBinding(t.Context(), edastore.Binding{Name: "n", Source: "sentry", Workflow: "implement"}) + if err != nil { + t.Fatalf("InsertBinding() error = %v", err) + } + b, err := st.GetBinding(t.Context(), id) + if err != nil { + t.Fatalf("GetBinding() error = %v", err) + } + if b.Status != edastore.StatusDraft { + t.Errorf("new binding status = %q, want %q: a binding must not be live before approval", b.Status, edastore.StatusDraft) + } +} + +// TestApproveRejectsUnknownBinding pins the sentinel the gRPC layer maps. +func TestApproveRejectsUnknownBinding(t *testing.T) { + st := newStore(t) + if err := st.ApproveBinding(t.Context(), "nosuchbinding"); !errors.Is(err, edastore.ErrBindingNotFound) { + t.Errorf("ApproveBinding(unknown) error = %v, want ErrBindingNotFound", err) + } +} + +// TestCaptureRoundTripsPayload pins the epic's core requirement: an event +// arrives with a payload nobody has modelled yet and must be readable back +// verbatim so an operator can map fields against it. +func TestCaptureRoundTripsPayload(t *testing.T) { + st := newStore(t) + body := `{"action":"created","issue":{"number":7,"title":"it broke"}}` + id, err := st.InsertCapture(t.Context(), edastore.Capture{ + Source: "sentry", Body: body, Headers: `{"X-Hook":"1"}`, ContentType: "application/json", + }) + if err != nil { + t.Fatalf("InsertCapture() error = %v", err) + } + got, err := st.Capture(t.Context(), id) + if err != nil { + t.Fatalf("Capture() error = %v", err) + } + if got.Body != body { + t.Errorf("Body round-trip = %q, want %q: the raw payload is what field mapping is designed against", got.Body, body) + } +} + +var _ = core.Collection{}