fix(captures): keep count-retention inside the inserted capture's org
This commit is contained in:
5 files changed
+117
-17
No files matched your search
@@ -1,7 +1,6 @@
|
|||||||
package postgres_test
|
package postgres_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -33,17 +32,22 @@ func TestListCapturesIsOrgScoped(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
ctx context.Context
|
orgID org.OrgID
|
||||||
want string
|
withOrg bool
|
||||||
|
want string
|
||||||
}{
|
}{
|
||||||
{"no org keeps the default org", ctx, sysID},
|
{"no org keeps the default org", "", false, sysID},
|
||||||
{"default org lists only its own", org.WithOrg(ctx, org.DefaultOrgID), sysID},
|
{"default org lists only its own", org.DefaultOrgID, true, sysID},
|
||||||
{"other org lists only its own", org.WithOrg(ctx, "acme"), acmeID},
|
{"other org lists only its own", "acme", true, acmeID},
|
||||||
}
|
}
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
got, err := eda.ListCaptures(tc.ctx, 10)
|
listCtx := ctx
|
||||||
|
if tc.withOrg {
|
||||||
|
listCtx = org.WithOrg(ctx, tc.orgID)
|
||||||
|
}
|
||||||
|
got, err := eda.ListCaptures(listCtx, 10)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
package postgres_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/source"
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||||
|
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
|
||||||
|
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Count retention keeps the newest N captures within the just-inserted
|
||||||
|
// capture's org, so one org's volume never evicts another org's captures.
|
||||||
|
func TestCaptureCountPruneStaysInOrg(t *testing.T) {
|
||||||
|
ctx := t.Context()
|
||||||
|
db := pgstore.Open(t)
|
||||||
|
eda := postgres.NewEDA(db.Pool, nil)
|
||||||
|
for _, owned := range []struct{ path, org string }{{"prune-a", "org-a"}, {"prune-b", "org-b"}} {
|
||||||
|
if err := eda.InsertSource(ctx, source.Source{Path: owned.path, Signing: source.SigningUnsigned}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Source ownership is set by another seam; write it directly.
|
||||||
|
if _, err := db.Pool.Exec(ctx, "UPDATE sources SET org_id=$1 WHERE path=$2", owned.org, owned.path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
base := time.Now().UTC().Truncate(time.Second)
|
||||||
|
insert := func(path string, at time.Time) string {
|
||||||
|
id, err := eda.InsertCapture(ctx, storecontract.CapturedEvent{Source: path, Body: "{}", ReceivedAt: at}, 0, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
// Quiet org first: its captures must survive the loud org's volume.
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
insert("prune-b", base.Add(time.Duration(i)*time.Second))
|
||||||
|
}
|
||||||
|
var loud []string
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
loud = append(loud, insert("prune-a", base.Add(time.Duration(10+i)*time.Second)))
|
||||||
|
}
|
||||||
|
count := func(owner string) int {
|
||||||
|
var n int
|
||||||
|
if err := db.Pool.QueryRow(ctx, "SELECT count(*) FROM captures WHERE org_id=$1", owner).Scan(&n); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
present := func(id string) bool {
|
||||||
|
var n int
|
||||||
|
if err := db.Pool.QueryRow(ctx, "SELECT count(*) FROM captures WHERE id=$1", id).Scan(&n); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return n == 1
|
||||||
|
}
|
||||||
|
if got := count("org-b"); got != 2 {
|
||||||
|
t.Fatalf("quiet org kept %d captures, want 2", got)
|
||||||
|
}
|
||||||
|
if got := count("org-a"); got != 3 {
|
||||||
|
t.Fatalf("loud org kept %d captures, want 3", got)
|
||||||
|
}
|
||||||
|
for _, id := range loud[:2] {
|
||||||
|
if present(id) {
|
||||||
|
t.Fatalf("loud org's oldest capture %s survived past the cap", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, id := range loud[2:] {
|
||||||
|
if !present(id) {
|
||||||
|
t.Fatalf("loud org's newest capture %s missing", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -107,20 +107,23 @@ func (s *EDA) InsertCapture(ctx context.Context, c storecontract.CapturedEvent,
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
return "", fmt.Errorf("edastore: insert capture: %w", err)
|
return "", fmt.Errorf("edastore: insert capture: %w", err)
|
||||||
}
|
}
|
||||||
if err := s.pruneCaptures(ctx, retention, maxEvents); err != nil {
|
if err := s.pruneCaptures(ctx, id, retention, maxEvents); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
return id, nil
|
return id, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *EDA) pruneCaptures(ctx context.Context, retention time.Duration, maxEvents int) error {
|
func (s *EDA) pruneCaptures(ctx context.Context, captureID string, retention time.Duration, maxEvents int) error {
|
||||||
if retention > 0 {
|
if retention > 0 {
|
||||||
if err := s.q.DeleteCapturesOlderThan(ctx, time.Now().Add(-retention).UTC()); err != nil {
|
if err := s.q.DeleteCapturesOlderThan(ctx, time.Now().Add(-retention).UTC()); err != nil {
|
||||||
return fmt.Errorf("edastore: prune captures by age: %w", err)
|
return fmt.Errorf("edastore: prune captures by age: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if maxEvents > 0 {
|
if maxEvents > 0 {
|
||||||
if err := s.q.DeleteCapturesBeyondCount(ctx, int32(maxEvents)); err != nil {
|
if err := s.q.DeleteCapturesBeyondCount(ctx, postgresdb.DeleteCapturesBeyondCountParams{
|
||||||
|
CaptureID: captureID,
|
||||||
|
MaxEvents: int32(maxEvents),
|
||||||
|
}); err != nil {
|
||||||
return fmt.Errorf("edastore: prune captures by count: %w", err)
|
return fmt.Errorf("edastore: prune captures by count: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -137,13 +137,25 @@ func (q *Queries) DeleteCaptureRefusalsBefore(ctx context.Context, refusedAt tim
|
|||||||
}
|
}
|
||||||
|
|
||||||
const deleteCapturesBeyondCount = `-- name: DeleteCapturesBeyondCount :exec
|
const deleteCapturesBeyondCount = `-- name: DeleteCapturesBeyondCount :exec
|
||||||
DELETE FROM captures WHERE id NOT IN (
|
DELETE FROM captures AS c
|
||||||
SELECT id FROM captures ORDER BY received_at DESC LIMIT $1
|
WHERE c.org_id = (SELECT m.org_id FROM captures AS m WHERE m.id = $1)
|
||||||
|
AND c.id NOT IN (
|
||||||
|
SELECT k.id FROM captures AS k
|
||||||
|
WHERE k.org_id = (SELECT m.org_id FROM captures AS m WHERE m.id = $1)
|
||||||
|
ORDER BY k.received_at DESC
|
||||||
|
LIMIT $2
|
||||||
)
|
)
|
||||||
`
|
`
|
||||||
|
|
||||||
func (q *Queries) DeleteCapturesBeyondCount(ctx context.Context, limit int32) error {
|
type DeleteCapturesBeyondCountParams struct {
|
||||||
_, err := q.db.Exec(ctx, deleteCapturesBeyondCount, limit)
|
CaptureID string
|
||||||
|
MaxEvents int32
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count retention stays within the just-inserted capture's org: one org's
|
||||||
|
// volume must never evict another org's recent captures.
|
||||||
|
func (q *Queries) DeleteCapturesBeyondCount(ctx context.Context, arg DeleteCapturesBeyondCountParams) error {
|
||||||
|
_, err := q.db.Exec(ctx, deleteCapturesBeyondCount, arg.CaptureID, arg.MaxEvents)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -30,8 +30,15 @@ LIMIT @entry_limit;
|
|||||||
DELETE FROM captures WHERE received_at < $1;
|
DELETE FROM captures WHERE received_at < $1;
|
||||||
|
|
||||||
-- name: DeleteCapturesBeyondCount :exec
|
-- name: DeleteCapturesBeyondCount :exec
|
||||||
DELETE FROM captures WHERE id NOT IN (
|
-- Count retention stays within the just-inserted capture's org: one org's
|
||||||
SELECT id FROM captures ORDER BY received_at DESC LIMIT $1
|
-- volume must never evict another org's recent captures.
|
||||||
|
DELETE FROM captures AS c
|
||||||
|
WHERE c.org_id = (SELECT m.org_id FROM captures AS m WHERE m.id = sqlc.arg(capture_id))
|
||||||
|
AND c.id NOT IN (
|
||||||
|
SELECT k.id FROM captures AS k
|
||||||
|
WHERE k.org_id = (SELECT m.org_id FROM captures AS m WHERE m.id = sqlc.arg(capture_id))
|
||||||
|
ORDER BY k.received_at DESC
|
||||||
|
LIMIT sqlc.arg(max_events)
|
||||||
);
|
);
|
||||||
|
|
||||||
-- name: InsertMapping :exec
|
-- name: InsertMapping :exec
|
||||||
|
|||||||
Reference in new issue
Block a user