feat(state): audit retention resource and observability sweep

This commit is contained in:
sam committed 2026-10-25 21:53:29 +11:00
commit 763053e698
9 files changed
+519 -2

No files matched your search

@@ -0,0 +1,103 @@
package controlplane
import (
"bytes"
"encoding/json"
"fmt"
"strconv"
"time"
"github.com/samcharles93/archie-core/internal/config"
)
const (
AuditRetentionKind = "audit-retention"
// MinimumAuditRetention is the shortest retention the resource accepts:
// below a week the audit trail is unlikely to outlive the investigation
// that needs it.
MinimumAuditRetention = 7 * 24 * time.Hour
// auditRetentionSeed keeps observability events for 90 days.
auditRetentionSeed = "2160h"
)
// auditRetention is the audit-retention document. Zero keeps the observability
// trail forever; any other value is a Go duration of at least a week.
type auditRetention struct {
RuntimeEventRetention string `json:"runtime_event_retention"`
}
func auditRetentionDefinition() Definition {
return Definition{
Kind: AuditRetentionKind, Title: "Audit retention", ApplyMode: "live",
Document: auditRetention{},
Seed: func(config.Config) any { return auditRetention{RuntimeEventRetention: auditRetentionSeed} },
Validate: validateAuditRetention,
Normalize: normalizeAuditRetention,
}
}
func validateAuditRetention(input []byte) error {
return validateAs(input, func(document auditRetention) error {
_, err := parseRuntimeEventRetention(document.RuntimeEventRetention)
return err
})
}
// RuntimeEventRetention decodes a stored audit-retention document and returns
// how long observability events are kept. Zero means keep forever. The sweep
// uses this same parser, so a value the control plane accepted is one the
// sweep can run.
func RuntimeEventRetention(document []byte) (time.Duration, error) {
decoded, err := auditRetentionDocument(document)
if err != nil {
return 0, err
}
return parseRuntimeEventRetention(decoded.RuntimeEventRetention)
}
func auditRetentionDocument(document []byte) (auditRetention, error) {
var decoded auditRetention
reader := json.NewDecoder(bytes.NewReader(document))
reader.DisallowUnknownFields()
if err := reader.Decode(&decoded); err != nil {
return auditRetention{}, fmt.Errorf("decode %s: %w", AuditRetentionKind, err)
}
return decoded, nil
}
// normalizeAuditRetention stores the retention in whole hours: the audit page
// parses "<n>h" only, so a value such as 604800s would read as zero there and
// the next save would disable pruning. Zero stays "0".
func normalizeAuditRetention(input []byte) ([]byte, error) {
decoded, err := auditRetentionDocument(input)
if err != nil {
return nil, err
}
retention, err := parseRuntimeEventRetention(decoded.RuntimeEventRetention)
if err != nil {
return nil, err
}
decoded.RuntimeEventRetention = formatRuntimeEventRetention(retention)
return json.Marshal(decoded)
}
func formatRuntimeEventRetention(retention time.Duration) string {
if retention == 0 {
return "0"
}
return strconv.FormatInt(int64(retention/time.Hour), 10) + "h"
}
func parseRuntimeEventRetention(value string) (time.Duration, error) {
retention, err := time.ParseDuration(value)
if err != nil {
return 0, fmt.Errorf("runtime_event_retention must be a duration such as 2160h: %w", err)
}
if retention < 0 {
return 0, fmt.Errorf("runtime_event_retention must not be negative")
}
if retention > 0 && retention < MinimumAuditRetention {
return 0, fmt.Errorf("runtime_event_retention must be at least 168h, or 0 to keep forever")
}
return retention, nil
}
@@ -64,6 +64,9 @@ func operationalDefinitions() []Definition {
// Live: every check and install reads the channel afresh.
{Kind: UpdateSettingsKind, Title: "Update settings", ApplyMode: "live", Document: controlplanerpc.UpdateSettings{}, Seed: func(config.Config) any { return controlplanerpc.UpdateSettings{Channel: "stable"} }, Validate: validateUpdateSettings},
{Kind: ContainerRuntimePoliciesKind, Title: "Container runtime policies", ApplyMode: "live", Document: containerRuntimePolicies{}, Seed: seedContainerPolicies, Validate: validateContainers, Normalize: normalizeContainerPolicies},
// Live: the retention sweep reads the document on every tick, so a
// change applies to the next sweep without a restart.
auditRetentionDefinition(),
}
}
+1
View File
@@ -182,6 +182,7 @@ var instanceOnlyKinds = map[string]bool{
TaskPolicyKind: true, ToolSettingsKind: true,
PluginSettingsKind: true, ContainerRuntimePoliciesKind: true, AgentProfileKind: true,
ExtensionSettingsKind: true, UpdateSettingsKind: true, WorkflowExecutionSettingsKind: true,
AuditRetentionKind: true,
}
// checkOrgWrite applies the rules a resource's static validation cannot: the
+3 -2
View File
@@ -73,10 +73,11 @@ func serviceAlert(name string) alert.Input {
}
}
// startSweeps starts the store's periodic work: auto-updating packages, and
// checking service health and pruning alerts.
// startSweeps starts the store's periodic work: auto-updating packages,
// checking service health and pruning alerts, and applying audit retention.
func (b *server) startSweeps(ctx context.Context, deps staterpc.Deps) {
go b.autoUpdatePackages(ctx, deps.Packages)
go b.sweepAuditRetention(ctx)
if deps.Alerts != nil {
go b.sweepAlerts(ctx, deps.Alerts)
}
+175
View File
@@ -0,0 +1,175 @@
package statestore
import (
"context"
"errors"
"slices"
"time"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
)
// auditRetentionSweepInterval is how often the stored retention is applied.
const auditRetentionSweepInterval = time.Hour
// auditEventPruner is the store's bounded, lock-guarded event delete.
type auditEventPruner interface {
PruneAuditEvents(ctx context.Context, kinds []string, cutoff time.Time) (int64, error)
}
// auditRetentionResources reads the instance's retention document.
type auditRetentionResources interface {
Resource(ctx context.Context, orgID, kind string) (storecontract.Resource, error)
}
// observabilityEventKinds is the hard-coded allowlist of high-volume event
// kinds the sweep may delete. It is code, not a setting: no stored value can
// widen it, and adding a kind here is the only way to make it prunable. Kinds
// a product view reads are deliberately absent.
func observabilityEventKinds() []string {
return []string{
events.KindStageStart,
events.KindAgentFinish,
events.KindToolCall,
events.KindLog,
events.KindTurnCompleted,
events.KindConfigCaptured,
events.KindCuratorRun,
events.KindCuratorAction,
events.KindCuratorError,
events.KindJobRun,
events.KindJobError,
}
}
// protectedEventKinds are the kinds the sweep must never delete: security
// relevant records (approvals and rejections, retries, cancellation and
// abandonment, pull-request actions, delivery records, binding and mapping
// changes, alerts, update reports) and the two observability kinds a product
// read depends on. stage_finish is the only source for StageStats' slowest
// stages and most failures; changes_captured is what api_tasks_changes rebuilds
// a task's captured-changes view from. Neither has a fallback, so pruning them
// would silently change what the product shows. Keeping the list here, rather
// than in a comment, is what lets the prunable predicate refuse a kind that
// appears in both lists.
func protectedEventKinds() []string {
return []string{
events.KindStageFinish,
events.KindChangesCaptured,
events.KindTaskQueued,
events.KindDeliveryRecorded,
events.KindParked,
events.KindOutcome,
events.KindChatOutcome,
events.KindPRMerged,
events.KindPRRejected,
events.KindPRClosed,
events.KindPRMergeRequested,
events.KindTaskDead,
events.KindOperatorApproved,
events.KindOperatorRejected,
events.KindAgentApproved,
events.KindAgentRejected,
events.KindTaskApproved,
events.KindTaskRejected,
events.KindOperatorRereviewed,
events.KindAgentRereviewed,
events.KindTaskRereviewed,
events.KindTaskRetried,
events.KindTaskCancelled,
events.KindTaskStopped,
events.KindTaskAbandoned,
events.KindRemediationRound,
events.KindReviewApproved,
events.KindStepRetried,
events.KindTaskWithdrawn,
events.KindTaskArchiveRequested,
events.KindWorkRequestSubmitted,
events.KindAlert,
events.KindBindingChanged,
events.KindMappingChanged,
events.KindBindingStarted,
events.KindWorkflowCallStarted,
events.KindIdentityStarted,
events.KindWorkflowCallFinished,
events.KindUpdateReport,
// Written by the daemon's binding dispatch as a bare literal.
"binding_dispatch_failure",
}
}
// prunableEventKinds is the allowlist with the protected set subtracted. The
// two are disjoint; the subtraction is the fail-safe, so a kind accidentally
// added to both is kept rather than deleted.
func prunableEventKinds() []string {
protected := protectedEventKinds()
return slices.DeleteFunc(slices.Clone(observabilityEventKinds()), func(kind string) bool {
return slices.Contains(protected, kind)
})
}
// prunableEventKind reports whether kind is on the sweep's allowlist.
func prunableEventKind(kind string) bool {
return slices.Contains(prunableEventKinds(), kind)
}
// sweepAuditRetention applies the audit-retention setting until ctx ends. It
// reads the resource on every tick, so an edit reaches the next sweep without
// a restart, and an unreadable or invalid document skips the tick rather than
// falling back to a default that prunes.
func (b *server) sweepAuditRetention(ctx context.Context) {
ticker := time.NewTicker(auditRetentionSweepInterval)
defer ticker.Stop()
for {
b.pruneAuditEvents(ctx)
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func (b *server) pruneAuditEvents(ctx context.Context) {
resources, ok := b.st.(auditRetentionResources)
if !ok {
return
}
resource, err := resources.Resource(ctx, storecontract.DefaultOrgID, controlplane.AuditRetentionKind)
if err != nil {
if ctx.Err() == nil {
b.log.Warn("audit retention not applied: setting unreadable, sweep skipped", "err", err)
}
return
}
retention, err := controlplane.RuntimeEventRetention(resource.Value)
if err != nil {
if ctx.Err() == nil {
b.log.Warn("audit retention not applied: setting invalid, sweep skipped", "err", err)
}
return
}
if retention == 0 {
// Keep forever: there is no cutoff to apply.
return
}
pruner, ok := b.st.(auditEventPruner)
if !ok {
return
}
cutoff := time.Now().Add(-retention)
rows, err := pruner.PruneAuditEvents(ctx, prunableEventKinds(), cutoff)
switch {
case errors.Is(err, postgres.ErrAuditPruneBusy):
b.log.Debug("audit retention sweep skipped: another sweep holds the lock")
case err != nil:
if ctx.Err() == nil {
b.log.Warn("prune audit events", "err", err)
}
default:
b.log.Info("pruned audit observability events", "rows", rows, "cutoff", cutoff.UTC())
}
}
@@ -0,0 +1,95 @@
package statestore
import (
"slices"
"testing"
"github.com/samcharles93/archie-core/internal/events"
)
func TestPrunableEventKindAllowsOnlyObservability(t *testing.T) {
tests := []struct {
kind string
want bool
}{
// The allowlist: high-volume observability only.
{events.KindStageStart, true},
{events.KindAgentFinish, true},
{events.KindToolCall, true},
{events.KindLog, true},
{events.KindTurnCompleted, true},
{events.KindConfigCaptured, true},
{events.KindCuratorRun, true},
{events.KindCuratorAction, true},
{events.KindCuratorError, true},
{events.KindJobRun, true},
{events.KindJobError, true},
// Approvals and rejections.
{events.KindOperatorApproved, false},
{events.KindAgentApproved, false},
{events.KindTaskApproved, false},
{events.KindOperatorRejected, false},
{events.KindAgentRejected, false},
{events.KindTaskRejected, false},
{events.KindPRRejected, false},
{events.KindReviewApproved, false},
// Retries.
{events.KindTaskRetried, false},
{events.KindStepRetried, false},
// Cancellation, stop, abandonment and withdrawal.
{events.KindTaskCancelled, false},
{events.KindTaskStopped, false},
{events.KindTaskAbandoned, false},
{events.KindTaskWithdrawn, false},
// Pull-request actions.
{events.KindPRMerged, false},
{events.KindPRClosed, false},
{events.KindPRMergeRequested, false},
// Delivery records.
{events.KindDeliveryRecorded, false},
// Binding and mapping changes.
{events.KindBindingChanged, false},
{events.KindMappingChanged, false},
{events.KindBindingStarted, false},
{"binding_dispatch_failure", false},
// Alerts and update reports.
{events.KindAlert, false},
{events.KindUpdateReport, false},
// Observability with a product read and no fallback: StageStats and
// the task's captured-changes view.
{events.KindStageFinish, false},
{events.KindChangesCaptured, false},
// Other lifecycle and operator actions stay too.
{events.KindTaskQueued, false},
{events.KindParked, false},
{events.KindOutcome, false},
{events.KindChatOutcome, false},
{events.KindTaskDead, false},
{events.KindOperatorRereviewed, false},
{events.KindAgentRereviewed, false},
{events.KindTaskRereviewed, false},
{events.KindRemediationRound, false},
{events.KindTaskArchiveRequested, false},
{events.KindWorkRequestSubmitted, false},
{events.KindWorkflowCallStarted, false},
{events.KindIdentityStarted, false},
{events.KindWorkflowCallFinished, false},
// A kind nobody registered is not prunable either.
{"unknown_kind", false},
}
for _, tt := range tests {
t.Run(tt.kind, func(t *testing.T) {
if got := prunableEventKind(tt.kind); got != tt.want {
t.Fatalf("prunableEventKind(%q) = %v, want %v", tt.kind, got, tt.want)
}
})
}
for _, kind := range protectedEventKinds() {
if slices.Contains(prunableEventKinds(), kind) {
t.Fatalf("protected kind %q is in the prunable allowlist", kind)
}
}
}
@@ -0,0 +1,82 @@
package postgres
import (
"context"
"errors"
"fmt"
"time"
"github.com/samcharles93/archie-core/internal/domain/access"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb"
)
const (
// auditPruneBatch is one delete's bound; auditPruneMaxBatches keeps a
// single sweep from owning the table for an unbounded time.
auditPruneBatch = 10_000
auditPruneMaxBatches = 100
auditRetentionActor = "archie-state-store"
auditRetentionSource = "audit retention"
)
// ErrAuditPruneBusy reports that another audit retention sweep holds the lock.
// The caller skips its tick rather than waiting.
var ErrAuditPruneBusy = errors.New("postgres: audit prune already running")
// auditPruneRecord is the value stored on the sweep's audit row.
type auditPruneRecord struct {
Cutoff time.Time `json:"cutoff"`
Rows int64 `json:"rows"`
}
// PruneAuditEvents deletes events of the given kinds older than cutoff in
// bounded batches and appends one sys_audit row recording the cutoff and the
// total removed, even when that total is zero. The whole sweep is one
// transaction under the audit prune lock, so the lock covers every batch and a
// concurrent sweep gets ErrAuditPruneBusy without touching a row. The kind
// list is the caller's allowlist: this method deletes nothing else.
func (s *Store) PruneAuditEvents(ctx context.Context, kinds []string, cutoff time.Time) (int64, error) {
if len(kinds) == 0 {
return 0, nil
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return 0, fmt.Errorf("prune audit events: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
locked, err := q.TryAuditPruneLock(ctx)
if err != nil {
return 0, fmt.Errorf("audit prune lock: %w", err)
}
if !locked {
return 0, ErrAuditPruneBusy
}
var total int64
for range auditPruneMaxBatches {
removed, err := q.PruneAuditEventBatch(ctx, postgresdb.PruneAuditEventBatchParams{
Kinds: kinds, Cutoff: cutoff.UTC(), Batch: auditPruneBatch,
})
if err != nil {
return 0, fmt.Errorf("prune audit events: %w", err)
}
total += removed
if removed < auditPruneBatch {
break
}
}
// The sweep is not an RPC, so it attributes its own row: the State Store
// pruned the events and the retention setting decided how many.
ctx = access.WithActor(access.WithSource(ctx, auditRetentionSource), auditRetentionActor)
if err := insertAuditValues(ctx, q, "audit_retention", "runtime_events", "pruned", nil,
auditPruneRecord{Cutoff: cutoff.UTC(), Rows: total}, 0, storecontract.DefaultOrgID); err != nil {
return 0, err
}
if err := tx.Commit(ctx); err != nil {
return 0, fmt.Errorf("prune audit events: %w", err)
}
return total, nil
}
@@ -167,3 +167,43 @@ func (q *Queries) LockAuditRecord(ctx context.Context, arg LockAuditRecordParams
_, err := q.db.Exec(ctx, lockAuditRecord, arg.TableName, arg.RecordKey)
return err
}
const pruneAuditEventBatch = `-- name: PruneAuditEventBatch :execrows
DELETE FROM events
WHERE id IN (
SELECT id FROM events
WHERE kind = ANY($1::text[])
AND at < $2::timestamptz
ORDER BY at
LIMIT $3::bigint
)
`
type PruneAuditEventBatchParams struct {
Kinds []string
Cutoff time.Time
Batch int64
}
// One bounded delete of the observability kinds the sweep allows, oldest
// first. The kind list is code, so no stored setting can widen it.
func (q *Queries) PruneAuditEventBatch(ctx context.Context, arg PruneAuditEventBatchParams) (int64, error) {
result, err := q.db.Exec(ctx, pruneAuditEventBatch, arg.Kinds, arg.Cutoff, arg.Batch)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const tryAuditPruneLock = `-- name: TryAuditPruneLock :one
SELECT pg_try_advisory_xact_lock(hashtextextended('archie.audit.prune', 0))
`
// The retention sweep's transaction-scoped lock. A second sweep skips its
// tick rather than queuing deletes behind this one.
func (q *Queries) TryAuditPruneLock(ctx context.Context) (bool, error) {
row := q.db.QueryRow(ctx, tryAuditPruneLock)
var pg_try_advisory_xact_lock bool
err := row.Scan(&pg_try_advisory_xact_lock)
return pg_try_advisory_xact_lock, err
}
@@ -25,6 +25,23 @@ VALUES (now(), sqlc.arg(table_name)::text, sqlc.arg(record_key)::text, sqlc.arg(
sqlc.arg(old_value)::jsonb, sqlc.arg(new_value)::jsonb, sqlc.arg(record_version)::bigint,
sqlc.arg(actor)::text, sqlc.arg(source)::text, sqlc.arg(request_id)::text, sqlc.arg(org_id)::text);
-- name: TryAuditPruneLock :one
-- The retention sweep's transaction-scoped lock. A second sweep skips its
-- tick rather than queuing deletes behind this one.
SELECT pg_try_advisory_xact_lock(hashtextextended('archie.audit.prune', 0));
-- name: PruneAuditEventBatch :execrows
-- One bounded delete of the observability kinds the sweep allows, oldest
-- first. The kind list is code, so no stored setting can widen it.
DELETE FROM events
WHERE id IN (
SELECT id FROM events
WHERE kind = ANY(sqlc.arg(kinds)::text[])
AND at < sqlc.arg(cutoff)::timestamptz
ORDER BY at
LIMIT sqlc.arg(batch)::bigint
);
-- name: AuditForRecords :many
SELECT id, at, table_name, record_key, field, old_value, new_value, record_version, actor, source, request_id
FROM sys_audit