Merge branch 'worktree-agent-a091696d945ed3377' into worktree-giggly-marinating-lamport

# Conflicts:
#	cmd/archie-state-store/recovery.go
This commit is contained in:
sam committed 2026-09-24 09:56:10 +10:00
commit a86be2fa49
17 files changed
+1391 -61

No files matched your search

+26 -19
View File
@@ -19,21 +19,31 @@ const recoveryUsage = `usage: archie-state-store <command> [flags]
With no command, archie-state-store serves the State Store gRPC contract.
The recovery commands operate on the task database file directly -- the
configured db_path with "-tasks.sqlite" appended -- and are how an operator
recovers a database the control plane will not start on. Stop the State Store
first: restore and rollback rewrite the file and refuse while another process
owns it, while backup takes a consistent snapshot of a serving store.
The recovery commands are how an operator recovers a database the control
plane will not start on. Without -db they operate on the PostgreSQL database
the configuration's database_url names, through pg_dump and pg_restore (which
must be on PATH); with -db, on a legacy SQLite task file (the configured
db_path with "-tasks.sqlite" appended).
backup -db FILE -out FILE write a snapshot of the database
restore -db FILE -from FILE replace the database with a snapshot
validate -db FILE [-config FILE] check the file, its stored settings and the
configuration the daemon would boot with
rollback -db FILE -kind KIND replay an earlier revision of a stored
backup takes a consistent snapshot of a serving database. restore is
destructive and offline: stop every archie service first, since it refuses
while any of them holds its ownership claim, and every write made after the
snapshot is lost. rollback refuses while the State Store serves. There is no
automatic schema rollback: the way back from a migration is restoring the
snapshot taken before it.
backup [-db FILE] -out FILE write a snapshot of the database
restore [-db FILE] -from FILE replace the database with a snapshot
validate [-db FILE] check the database, its stored settings and
the configuration the daemon would boot with
rollback [-db FILE] -kind KIND replay an earlier revision of a stored
[-revision N] resource through the ordinary replace
import [-db-path PATH] copy the legacy SQLite stores named from
[-database-url URL] db_path into an empty Postgres database,
[-config FILE] once; refuses while any Archie service runs
once; refuses while any Archie service runs
Every command takes -config FILE and -config-overlay FILE: the configuration
the daemon boots with, which names the database.
`
// runRecovery parses and performs one offline recovery command, reporting the
@@ -58,22 +68,19 @@ func runRecovery(args []string, stdout, stderr io.Writer) int {
flags := flag.NewFlagSet("archie-state-store "+command, flag.ContinueOnError)
flags.SetOutput(stderr)
flags.Usage = func() { fmt.Fprint(stderr, recoveryUsage) }
flags.StringVar(&options.DB, "db", "", "task database file the State Store owns (<db_path>-tasks.sqlite)")
flags.StringVar(&options.DB, "db", "", "legacy SQLite task file (<db_path>-tasks.sqlite); omit to use the configured database_url")
flags.StringVar(&options.Out, "out", "", "snapshot file backup writes")
flags.StringVar(&options.From, "from", "", "snapshot file restore reads")
flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays")
flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)")
// validate asks a question about the process rather than the file, and
// import reads db_path and database_url from it, so only those two take
// the configuration the process would boot with.
if command == archied.RecoveryImport {
flags.StringVar(&options.DBPath, "db-path", "", "configured db_path the legacy files are named from (default: the configuration's)")
flags.StringVar(&options.DatabaseURL, "database-url", "", "Postgres URL to import into (default: the configuration's database_url)")
}
if command == archied.RecoveryValidate || command == archied.RecoveryImport {
flags.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory the daemon boots with")
flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
}
// The configuration names the PostgreSQL database, and validate also checks
// the stored settings against it.
flags.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory the daemon boots with")
flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
if err := flags.Parse(args[1:]); err != nil {
// A requested help is not a wrong command line: the usage above is the
// answer, and the serve path exits 0 for the same request.
@@ -0,0 +1,43 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
// Without -db every recovery command targets the configured PostgreSQL
// database, so each one accepts -config and refuses, naming the key, when the
// configuration names no database.
func TestRecoveryWithoutDBUsesTheConfiguredDatabase(t *testing.T) {
dir := t.TempDir()
configPath := writeMinimalConfig(t, dir)
raw, err := os.ReadFile(configPath)
if err != nil {
t.Fatalf("read config: %v", err)
}
var kept []string
for line := range strings.Lines(string(raw)) {
if !strings.HasPrefix(line, "database_url") {
kept = append(kept, line)
}
}
if err := os.WriteFile(configPath, []byte(strings.Join(kept, "")), 0o600); err != nil {
t.Fatalf("write config: %v", err)
}
tests := [][]string{
{"backup", "-out", filepath.Join(dir, "snapshot.dump")},
{"restore", "-from", filepath.Join(dir, "snapshot.dump")},
{"validate"},
{"rollback", "-kind", "settings"},
}
for _, args := range tests {
t.Run(args[0], func(t *testing.T) {
code, _, stderr := runRecoveryCmd(t, append(args, "-config", configPath)...)
if code != 1 || !strings.Contains(stderr, "database_url") {
t.Fatalf("%s exited %d, stderr %q; want 1 naming database_url", args[0], code, stderr)
}
})
}
}
@@ -165,6 +165,31 @@ own `sqlite3` shell, so the installer and an operator run one implementation.
blunt one, and is also the way back from a schema migration a release cannot
grow out of.
### The same commands on PostgreSQL
Without `-db`, each command works on the PostgreSQL database the configuration's
`database_url` names (`internal/app/archied/state_store_recovery_postgres.go`,
`internal/infrastructure/postgres/maintenance.go`). One database backs every
service, so the scope is always the whole database.
- `backup -out F` runs `pg_dump --format=custom`, reads the archive back with
`pg_restore --list` and only then replaces `F`. It takes no claim and runs
against a serving database.
- `restore -from F` is destructive and offline. It claims every service role's
ownership lock (`state-store` and `gateway`) without waiting and refuses if
any is held or the check cannot complete, then drops tables the snapshot does
not hold and runs `pg_restore --clean --single-transaction`. Every write made
after the snapshot is lost.
- `validate` refuses a schema newer than the binary's highest migration, then
runs the same two validation layers.
- `rollback` holds the `state-store` claim for the replay.
There is no automatic schema rollback. goose `Down` drops tables, so it is data
loss, not a rollback. `scripts/archie-update-install` refuses a daemon update
it cannot snapshot first, and on a failed update `scripts/archie-update-watchdog`
rolls the binaries back, leaves the database alone and prints the `restore`
command for the pre-update snapshot. Whether to run it is the operator's call.
## Gateway restart: two constraints learned the hard way
Telegram `/restart` (`internal/channels/telegram/restart.go` plus the `Start`
+1 -1
View File
@@ -100,7 +100,7 @@
"url": "/dev/architecture/safe-change-and-recovery/",
"title": "Safe Change and Recovery",
"section": "Architecture",
"body": "# Safe Change and Recovery\n\n**Status:** Requirements approved; mechanics are under design. Database-backed settings are implemented through the control plane (2026-09): see the degrade/recovery section below, which records a gap rather than a solution. The full runtime-supervision protocol (bounded observation, versioned audit trail) remains under design.\n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Purpose\n\nArchie is built and improved by agents as well as humans. The architecture MUST\nmake a change easy to locate, understand, test, audit, extend, and reverse\nwithout requiring its author to understand unrelated parts of the application.\n\nSeparation of concerns is therefore an operational safety boundary, not only a\ncode-organisation preference.\n\n## Required outcomes\n\n- An agent can improve a focused feature without receiving authority over\n unrelated domains.\n- Every event, function, action, setting, and policy has an identifiable owner\n and provenance.\n- Source and runtime changes pass deterministic checks outside the actor making\n the change.\n- A rejected or unhealthy candidate cannot replace the last known working\n version.\n- Invalid user configuration does not terminate an otherwise healthy\n application.\n- Configuration and service failures are observable, attributed, and eligible\n for bounded remediation or rollback.\n- The application continues serving unaffected capabilities when one\n capability is misconfigured or unhealthy.\n- Recovery failure is reported and contained rather than hidden in an\n unbounded retry loop.\n\n## Source changes\n\nAgent-authored source changes occur through the task-execution boundary in an\nisolated workspace. The agent does not directly mutate the running\ninstallation.\n\nPromotion is controlled by deterministic repository-owned policy, including\ntests, generated-reference drift checks, protected paths, change scope, and\nother required gates. The actor proposing a change cannot waive these checks.\n\nRepository and domain separation SHOULD allow a task to load and modify only\nthe governing requirements and code for its declared scope.\n\n## Runtime changes\n\nConfiguration and other runtime changes are proposed as candidates rather than\nmutating the active state in place.\n\nThe affected domain owns:\n\n- the command describing the semantic change;\n- validation of its typed settings;\n- health signals that demonstrate correct operation;\n- domain-specific remediation and rollback consequences.\n\nShared policy supplies consistent evaluation, evidence, and audit mechanics.\nIdentity supplies actor attribution. Infrastructure supplies external\nconfiguration storage, service control, and other operational adapters.\n\n## Runtime change protocol\n\nLast-known-good preservation and candidate promotion are the universal runtime\nchange protocol.\n\n1. An identity proposes an immutable candidate version.\n2. Infrastructure parses it without changing active state.\n3. Each affected domain validates its typed settings.\n4. Shared policy evaluates cross-cutting constraints.\n5. Each affected capability prepares the candidate without destroying its\n active instance.\n6. Capability-specific health checks prove the candidate works.\n7. Runtime supervision atomically promotes the candidate.\n8. A bounded observation period watches for delayed failure.\n9. Failure restores the last-known-good version and records the reason,\n evidence, actor, and involved versions.\n10. Repeated remediation failure stops retrying, isolates the capability, and\n reports the fault.\n\nThe affected domain owns the semantic change command, typed validation, health\nsignals, and remediation consequences. Shared runtime supervision owns the\npromotion transaction. Infrastructure performs storage and service-control\noperations.\n\nThe protocol's exact Go contracts, storage model, atomicity boundary, and\nobservation rules require a focused runtime-supervision design pass.\n\n## Failure isolation\n\nInvalid input MUST leave the active valid configuration unchanged and produce a\ndiagnostic record.\n\nA capability that cannot start or reload SHOULD fail independently where its\nabsence does not violate a declared application invariant. Unaffected\ncapabilities continue operating.\n\nRemediation MUST be bounded, observable, and policy-driven. Recovery MUST NOT\nsilently oscillate between versions or retry forever.\n\n## Database settings: degrade paths and recovery (2026-09)\n\nThe control plane owns runtime-tunable settings and stores them in `archie.db`.\nThis replaced a dashboard overlay store that degraded to file config on any\nfailure and had three documented ways back. The replacement has none, and the\nposture is the opposite: **the daemon fails closed.**\n\n### Degrade paths\n\n| Failure | Boot behaviour | Where the operator sees it |\n| --- | --- | --- |\n| State Store uLine truncated
"body": "# Safe Change and Recovery\n\n**Status:** Requirements approved; mechanics are under design. Database-backed settings are implemented through the control plane (2026-09): see the degrade/recovery section below, which records a gap rather than a solution. The full runtime-supervision protocol (bounded observation, versioned audit trail) remains under design.\n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Purpose\n\nArchie is built and improved by agents as well as humans. The architecture MUST\nmake a change easy to locate, understand, test, audit, extend, and reverse\nwithout requiring its author to understand unrelated parts of the application.\n\nSeparation of concerns is therefore an operational safety boundary, not only a\ncode-organisation preference.\n\n## Required outcomes\n\n- An agent can improve a focused feature without receiving authority over\n unrelated domains.\n- Every event, function, action, setting, and policy has an identifiable owner\n and provenance.\n- Source and runtime changes pass deterministic checks outside the actor making\n the change.\n- A rejected or unhealthy candidate cannot replace the last known working\n version.\n- Invalid user configuration does not terminate an otherwise healthy\n application.\n- Configuration and service failures are observable, attributed, and eligible\n for bounded remediation or rollback.\n- The application continues serving unaffected capabilities when one\n capability is misconfigured or unhealthy.\n- Recovery failure is reported and contained rather than hidden in an\n unbounded retry loop.\n\n## Source changes\n\nAgent-authored source changes occur through the task-execution boundary in an\nisolated workspace. The agent does not directly mutate the running\ninstallation.\n\nPromotion is controlled by deterministic repository-owned policy, including\ntests, generated-reference drift checks, protected paths, change scope, and\nother required gates. The actor proposing a change cannot waive these checks.\n\nRepository and domain separation SHOULD allow a task to load and modify only\nthe governing requirements and code for its declared scope.\n\n## Runtime changes\n\nConfiguration and other runtime changes are proposed as candidates rather than\nmutating the active state in place.\n\nThe affected domain owns:\n\n- the command describing the semantic change;\n- validation of its typed settings;\n- health signals that demonstrate correct operation;\n- domain-specific remediation and rollback consequences.\n\nShared policy supplies consistent evaluation, evidence, and audit mechanics.\nIdentity supplies actor attribution. Infrastructure supplies external\nconfiguration storage, service control, and other operational adapters.\n\n## Runtime change protocol\n\nLast-known-good preservation and candidate promotion are the universal runtime\nchange protocol.\n\n1. An identity proposes an immutable candidate version.\n2. Infrastructure parses it without changing active state.\n3. Each affected domain validates its typed settings.\n4. Shared policy evaluates cross-cutting constraints.\n5. Each affected capability prepares the candidate without destroying its\n active instance.\n6. Capability-specific health checks prove the candidate works.\n7. Runtime supervision atomically promotes the candidate.\n8. A bounded observation period watches for delayed failure.\n9. Failure restores the last-known-good version and records the reason,\n evidence, actor, and involved versions.\n10. Repeated remediation failure stops retrying, isolates the capability, and\n reports the fault.\n\nThe affected domain owns the semantic change command, typed validation, health\nsignals, and remediation consequences. Shared runtime supervision owns the\npromotion transaction. Infrastructure performs storage and service-control\noperations.\n\nThe protocol's exact Go contracts, storage model, atomicity boundary, and\nobservation rules require a focused runtime-supervision design pass.\n\n## Failure isolation\n\nInvalid input MUST leave the active valid configuration unchanged and produce a\ndiagnostic record.\n\nA capability that cannot start or reload SHOULD fail independently where its\nabsence does not violate a declared application invariant. Unaffected\ncapabilities continue operating.\n\nRemediation MUST be bounded, observable, and policy-driven. Recovery MUST NOT\nsilently oscillate between versions or retry forever.\n\n## Database settings: degrade paths and recovery (2026-09)\n\nThe control plane owns runtime-tunable settings and stores them in `archie.db`.\nThis replaced a dashboard overlay store that degraded to file config on any\nfailure and had three documented ways back. The replacement has none, and the\nposture is the opposite: **the daemon fails closed.**\n\n### Degrade paths\n\n| Failure | Boot behaviour | Where the operator sees it |\n| --- | --- | --- |\n| State Store uLine truncated
},
{
"url": "/dev/prds/01-project-management/",
+1 -1
View File
@@ -35,7 +35,7 @@ func (b *boot) openStateStorePool(ctx context.Context) error {
b.log.Error("migrate state store postgres schema", "err", err)
return err
}
ownership, err := postgres.AcquireOwnership(ctx, pool, "state-store")
ownership, err := postgres.AcquireOwnership(ctx, pool, postgres.OwnerStateStore)
if err != nil {
pool.Close()
b.log.Error("claim state store ownership", "err", err)
+54 -32
View File
@@ -5,12 +5,12 @@
// State Store is up -- is unavailable in exactly the case that needs it
// (docs/architecture/safe-change-and-recovery.md).
//
// Every operation works on the task database file directly, with the State
// Store stopped, and the ones that write refuse when another process owns the
// file. Backup is the exception and deliberately so: the update installer
// cannot stop the process that runs it, so the snapshot that makes a failed
// update reversible is taken against a serving store through SQLite's own
// consistent copy.
// With -db an operation works on the legacy SQLite task file; without it, on
// the PostgreSQL database database_url names (state_store_recovery_postgres.go).
// The ones that write refuse while a serving process owns the store. Backup is
// the exception and deliberately so: the update installer cannot stop the
// process that runs it, so the snapshot that makes a failed update recoverable
// is taken against a serving store through a consistent copy.
//
// The control-plane server both validate and rollback read through is built by
// openStateStoreControlPlane -- the same root helper the served path uses -- so
@@ -25,6 +25,7 @@ import (
"fmt"
"time"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/config"
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
@@ -82,30 +83,35 @@ type StateStoreRecoveryOptions struct {
// RunStateStoreRecovery performs one offline operation on the task database and
// returns the line the command reports on stdout.
func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
// import names its legacy sources from db_path and its target from
// database_url itself.
if options.Operation == RecoveryImport {
return importLegacy(ctx, options)
}
// -db names the legacy SQLite task file; without it the command works on
// the PostgreSQL database the configuration names.
if options.DB == "" {
return runPostgresRecovery(ctx, options)
}
switch options.Operation {
case RecoveryBackup:
if options.DB == "" || options.Out == "" {
return "", errors.New("backup requires -db and -out")
if options.Out == "" {
return "", errors.New("backup requires -out")
}
return summarise(store.Backup(ctx, options.DB, options.Out), "backed up %s to %s", options.DB, options.Out)
case RecoveryRestore:
if options.DB == "" || options.From == "" {
return "", errors.New("restore requires -db and -from")
if options.From == "" {
return "", errors.New("restore requires -from")
}
return summarise(store.Restore(ctx, options.DB, options.From),
"restored %s from %s; start the State Store again", options.DB, options.From)
case RecoveryValidate:
if options.DB == "" {
return "", errors.New("validate requires -db")
}
return validateStore(ctx, options)
case RecoveryRollback:
if options.DB == "" || options.Kind == "" {
return "", errors.New("rollback requires -db and -kind")
if options.Kind == "" {
return "", errors.New("rollback requires -kind")
}
return rollbackResource(ctx, options)
case RecoveryImport:
return importLegacy(ctx, options)
default:
return "", fmt.Errorf("unknown recovery command %q", options.Operation)
}
@@ -157,34 +163,45 @@ func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (stri
// with the writer's validator, and boot's gate is the config check alone.
// That is the same stance the layering takes for a kind the table is missing:
// what the daemon reads for a kind nothing was ever stored for is the seed.
checked := 0
stored, err := st.StoresResources(ctx)
if err != nil {
return "", err
}
if stored {
if checked, err = server.ValidateStored(ctx); err != nil {
return "", err
}
}
base, err := bootConfig(ctx, options)
checked, err := checkBootGate(ctx, server, stored, options)
if err != nil {
return "", err
}
return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", options.DB, version, checked), nil
}
// checkBootGate runs the writer's per-resource validation over what is stored,
// then boot's gate over the file config with the stored resources layered on,
// and returns how many stored resources validated.
func checkBootGate(ctx context.Context, server *controlplane.Server, stored bool, options StateStoreRecoveryOptions) (int, error) {
checked := 0
if stored {
var err error
if checked, err = server.ValidateStored(ctx); err != nil {
return 0, err
}
}
base, err := bootConfig(ctx, options)
if err != nil {
return 0, err
}
document := base
if stored {
document, _, err = server.StoredRuntimeConfig(ctx, base)
if err != nil {
return "", err
return 0, err
}
}
if err := configuration.Validate(&document); err != nil {
// The daemon's own wording, so an operator can match this failure to the
// one that stopped archied starting.
return "", fmt.Errorf("validate database settings: %w", err)
return 0, fmt.Errorf("validate database settings: %w", err)
}
return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", options.DB, version, checked), nil
return checked, nil
}
// bootConfig resolves the configuration the daemon would boot with. validate is
@@ -228,22 +245,27 @@ func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (s
return "", err
}
defer func() { _ = st.Close() }()
return replayRevision(ctx, st, options)
}
server, err := openStateStoreControlPlane(st)
// replayRevision puts back the revision options names through the control
// plane's ordinary replace. The caller holds the store's ownership claim.
func replayRevision(ctx context.Context, resources controlplane.ResourceStore, options StateStoreRecoveryOptions) (string, error) {
server, err := openStateStoreControlPlane(resources)
if err != nil {
return "", err
}
if !server.Owns(options.Kind) {
return "", fmt.Errorf("unknown resource kind %q", options.Kind)
}
current, err := st.Resource(ctx, options.Kind)
current, err := resources.Resource(ctx, options.Kind)
if errors.Is(err, store.ErrResourceNotFound) {
return "", fmt.Errorf("%s has no stored value to roll back", options.Kind)
}
if err != nil {
return "", fmt.Errorf("read %s: %w", options.Kind, err)
}
revision, err := revisionToReplay(ctx, st, options.Kind, current.Version, options.Revision)
revision, err := revisionToReplay(ctx, resources, options.Kind, current.Version, options.Revision)
if err != nil {
return "", err
}
@@ -265,7 +287,7 @@ func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (s
// revisionToReplay picks the value to put back: the revision the operator
// named, or the newest one older than the value the resource carries now.
func revisionToReplay(ctx context.Context, st *store.Store, kind string, current, requested int64) (store.Resource, error) {
func revisionToReplay(ctx context.Context, st controlplane.ResourceStore, kind string, current, requested int64) (store.Resource, error) {
history, err := st.ResourceHistory(ctx, kind, 0)
if err != nil {
return store.Resource{}, fmt.Errorf("read %s history: %w", kind, err)
@@ -0,0 +1,108 @@
package archied
import (
"context"
"errors"
"fmt"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
)
// runPostgresRecovery performs one recovery operation on the PostgreSQL
// database the configuration names. There is no automatic schema rollback:
// goose Down drops tables, so the way back from a migration is restoring the
// snapshot taken before it, which loses every write made after that snapshot.
func runPostgresRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
cfg, err := bootConfig(ctx, options)
if err != nil {
return "", err
}
url := cfg.DatabaseURL
if url == "" {
return "", errors.New("database_url is required: the configuration names no PostgreSQL database (pass -db for a legacy SQLite task file)")
}
switch options.Operation {
case RecoveryBackup:
if options.Out == "" {
return "", errors.New("backup requires -out")
}
if err := postgres.Backup(ctx, url, options.Out); err != nil {
return "", err
}
return "backed up the whole database to " + options.Out, nil
case RecoveryRestore:
if options.From == "" {
return "", errors.New("restore requires -from")
}
if err := postgres.Restore(ctx, url, options.From); err != nil {
return "", err
}
return fmt.Sprintf("restored the whole database from %s; every write made after the snapshot is gone; start the archie services again", options.From), nil
case RecoveryValidate:
return validatePostgres(ctx, url, options)
case RecoveryRollback:
if options.Kind == "" {
return "", errors.New("rollback requires -kind")
}
return rollbackPostgres(ctx, url, options)
default:
return "", fmt.Errorf("unknown recovery command %q", options.Operation)
}
}
// openRecoveryPool opens the database without migrating it: recovery inspects
// or repairs the schema it finds, and refuses one newer than this binary.
func openRecoveryPool(ctx context.Context, url string) (*pgxpool.Pool, int64, error) {
pool, err := postgres.Open(ctx, url)
if err != nil {
return nil, 0, err
}
version, err := postgres.SchemaVersion(ctx, pool)
if err == nil {
err = postgres.CheckSchemaVersion(version)
}
if err != nil {
pool.Close()
return nil, 0, err
}
return pool, version, nil
}
// validatePostgres is validateStore for the PostgreSQL database. An
// unmigrated database holds nothing to check; the State Store migrates and
// seeds it on its next start, so only boot's config gate applies.
func validatePostgres(ctx context.Context, url string, options StateStoreRecoveryOptions) (string, error) {
pool, version, err := openRecoveryPool(ctx, url)
if err != nil {
return "", err
}
defer pool.Close()
server, err := openStateStoreControlPlane(postgres.NewResources(pool))
if err != nil {
return "", err
}
checked, err := checkBootGate(ctx, server, version > 0, options)
if err != nil {
return "", err
}
return fmt.Sprintf("the database is valid at schema version %d; %d stored resources validate", version, checked), nil
}
// rollbackPostgres replays an earlier revision holding the State Store's
// ownership claim, so it refuses while a State Store serves and none can
// start until it finishes.
func rollbackPostgres(ctx context.Context, url string, options StateStoreRecoveryOptions) (string, error) {
pool, _, err := openRecoveryPool(ctx, url)
if err != nil {
return "", err
}
defer pool.Close()
claim, err := postgres.AcquireOwnership(ctx, pool, postgres.OwnerStateStore)
if err != nil {
return "", fmt.Errorf("rollback refused, stop the State Store first: %w", err)
}
defer func() { _ = claim.Release(context.WithoutCancel(ctx)) }()
return replayRevision(ctx, postgres.NewResources(pool), options)
}
@@ -0,0 +1,201 @@
package archied
import (
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
)
// pgRecoveryFixture is a migrated database and the config file naming it,
// which is all the Postgres recovery path is given: it resolves database_url
// from the configuration exactly as the State Store does.
func pgRecoveryFixture(t *testing.T) (configPath string, pool *pgxpool.Pool, url string) {
t.Helper()
url = pgtest.URL(t)
dir := t.TempDir()
configPath = filepath.Join(dir, "config.toml")
body := fmt.Sprintf("bot_user = 'archie'\ndb_path = %q\ndatabase_url = %q\n[forge]\ntype = 'github'\nhost = 'https://github.example.com'\n",
filepath.Join(dir, "archie.db"), url)
if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
pool, err := postgres.Open(t.Context(), url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(pool.Close)
if err := postgres.Migrate(t.Context(), pool, postgres.Migrations()); err != nil {
t.Fatal(err)
}
return configPath, pool, url
}
func requirePGTools(t *testing.T) {
t.Helper()
for _, tool := range []string{"pg_dump", "pg_restore"} {
if _, err := exec.LookPath(tool); err != nil {
t.Skipf("%s is not on PATH; this test needs the PostgreSQL 18 client tools", tool)
}
}
}
func putResource(t *testing.T, pool *pgxpool.Pool, kind, value, label string) {
t.Helper()
resources := postgres.NewResources(pool)
expected := int64(0)
if current, err := resources.Resource(t.Context(), kind); err == nil {
expected = current.Version
}
if _, err := resources.PutResource(t.Context(), storecontract.ResourceWrite{
Kind: kind, Value: []byte(value), Actor: label, Source: "test",
RequestID: fmt.Sprintf("%s-%d", label, expected+1), ExpectedVersion: expected,
}); err != nil {
t.Fatalf("put %s: %v", kind, err)
}
}
func holdRole(t *testing.T, url, role string) {
t.Helper()
pool, err := postgres.Open(t.Context(), url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(pool.Close)
claim, err := postgres.AcquireOwnership(t.Context(), pool, role)
if err != nil {
t.Fatalf("hold %s: %v", role, err)
}
t.Cleanup(func() { _ = claim.Release(t.Context()) })
}
func TestPostgresRecoveryRequiresADatabaseURL(t *testing.T) {
dir := t.TempDir()
configPath := filepath.Join(dir, "config.toml")
if err := os.WriteFile(configPath, []byte("bot_user = 'archie'\n[forge]\ntype = 'github'\nhost = 'https://github.example.com'\n"), 0o600); err != nil {
t.Fatal(err)
}
for _, op := range []string{RecoveryBackup, RecoveryRestore, RecoveryValidate, RecoveryRollback} {
t.Run(op, func(t *testing.T) {
_, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{
Operation: op, Config: configPath, Out: filepath.Join(dir, "o"), From: filepath.Join(dir, "f"), Kind: "k",
})
if err == nil || !strings.Contains(err.Error(), "database_url") {
t.Fatalf("%s without database_url = %v, want a refusal naming database_url", op, err)
}
})
}
}
func TestPostgresRecoveryBackupAndRestore(t *testing.T) {
requirePGTools(t)
configPath, pool, url := pgRecoveryFixture(t)
putResource(t, pool, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "snapshot")
snapshot := filepath.Join(t.TempDir(), "snapshot.dump")
if _, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryBackup, Config: configPath, Out: snapshot}); err != nil {
t.Fatalf("backup: %v", err)
}
putResource(t, pool, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "later")
for _, role := range postgres.ServiceOwners {
t.Run("refused while "+role+" serves", func(t *testing.T) {
holdRole(t, url, role)
_, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryRestore, Config: configPath, From: snapshot})
if !errors.Is(err, postgres.ErrOwned) {
t.Fatalf("restore with %s serving = %v, want ErrOwned", role, err)
}
})
}
summary, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryRestore, Config: configPath, From: snapshot})
if err != nil {
t.Fatalf("restore: %v", err)
}
if !strings.Contains(summary, "after the snapshot") {
t.Errorf("restore must say writes after the snapshot are lost; summary = %q", summary)
}
current, err := postgres.NewResources(pool).Resource(t.Context(), controlplane.ModelRoleAssignmentsKind)
if err != nil {
t.Fatal(err)
}
if current.Version != 1 {
t.Errorf("restored version = %d, want the snapshot's 1", current.Version)
}
}
func TestPostgresRecoveryValidate(t *testing.T) {
tests := []struct {
name string
setup func(t *testing.T, pool *pgxpool.Pool)
wantErr string
}{
{name: "healthy store", setup: func(*testing.T, *pgxpool.Pool) {}},
{name: "schema newer than this binary", wantErr: "newer than this binary", setup: func(t *testing.T, pool *pgxpool.Pool) {
if _, err := pool.Exec(t.Context(), "INSERT INTO goose_db_version (version_id, is_applied) VALUES ($1, true)", postgres.SupportedSchemaVersion()+1); err != nil {
t.Fatal(err)
}
}},
{name: "stored value boot refuses", wantErr: "dispatch.trigger", setup: func(t *testing.T, pool *pgxpool.Pool) {
putResource(t, pool, "scheduling-policy", `{"poll_interval":"1m","max_retries":3,"dispatch":{"trigger":"bogus"}}`, "bad")
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
configPath, pool, _ := pgRecoveryFixture(t)
tt.setup(t, pool)
summary, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryValidate, Config: configPath})
if tt.wantErr == "" {
if err != nil {
t.Fatalf("validate: %v", err)
}
if !strings.Contains(summary, fmt.Sprintf("schema version %d", postgres.SupportedSchemaVersion())) {
t.Errorf("summary = %q, want the schema version", summary)
}
return
}
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("validate = %v, want refusal containing %q", err, tt.wantErr)
}
})
}
}
func TestPostgresRecoveryRollback(t *testing.T) {
configPath, pool, url := pgRecoveryFixture(t)
kind := controlplane.ModelRoleAssignmentsKind
putResource(t, pool, kind, `{"implement":"anthropic/claude"}`, "first")
putResource(t, pool, kind, `{"implement":"openai/gpt"}`, "second")
options := StateStoreRecoveryOptions{Operation: RecoveryRollback, Config: configPath, Kind: kind}
t.Run("refused while the State Store serves", func(t *testing.T) {
holdRole(t, url, postgres.OwnerStateStore)
if _, err := RunStateStoreRecovery(t.Context(), options); !errors.Is(err, postgres.ErrOwned) {
t.Fatalf("rollback with the State Store serving = %v, want ErrOwned", err)
}
})
if _, err := RunStateStoreRecovery(t.Context(), options); err != nil {
t.Fatalf("rollback: %v", err)
}
current, err := postgres.NewResources(pool).Resource(t.Context(), kind)
if err != nil {
t.Fatal(err)
}
if string(current.Value) != `{"implement": "anthropic/claude"}` && string(current.Value) != `{"implement":"anthropic/claude"}` {
t.Errorf("value = %s, want the first revision's", current.Value)
}
if current.Version != 3 {
t.Errorf("version = %d, want the rollback recorded as revision 3", current.Version)
}
}
@@ -9,14 +9,6 @@ import (
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb"
)
// Service owner names: every Archie process that serves this database holds
// one of these claims for its whole life, so an offline rewrite checks them
// all.
const (
OwnerStateStore = "state-store"
OwnerGateway = "gateway"
)
// ImportComplete reports whether the one-time legacy import committed and
// verified against this migrated database. Serving over non-empty legacy
// files is refused without it.
@@ -0,0 +1,258 @@
package postgres
import (
"bytes"
"context"
"errors"
"fmt"
"io/fs"
"net/url"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/pressly/goose/v3"
)
// ServiceOwners is every serving role that reads or writes the database. A
// destructive offline operation must hold all of them: one database backs
// every service, so the State Store being stopped alone proves nothing about
// the Gateway.
var ServiceOwners = []string{OwnerStateStore, OwnerGateway}
// ErrSchemaTooNew is returned for a database migrated by a newer release than
// this binary. goose Down is not a rollback (it drops tables), so the only way
// back is restoring the snapshot taken before that migration.
var ErrSchemaTooNew = errors.New("postgres: database schema is newer than this binary supports")
// gooseTable is goose's version table; its presence is what marks an archive
// as a dump of an archie database rather than of anything pg_restore can read.
const gooseTable = "public.goose_db_version"
// SupportedSchemaVersion is the highest migration this binary embeds.
func SupportedSchemaVersion() int64 {
names, err := fs.Glob(Migrations(), "*.sql")
if err != nil {
// Unreachable: the pattern is well formed.
panic(err)
}
var highest int64
for _, name := range names {
if version, err := goose.NumericComponent(name); err == nil {
highest = max(highest, version)
}
}
return highest
}
// CheckSchemaVersion refuses a schema this binary cannot serve.
func CheckSchemaVersion(version int64) error {
if supported := SupportedSchemaVersion(); version > supported {
return fmt.Errorf("%w (database at %d, binary supports %d)", ErrSchemaTooNew, version, supported)
}
return nil
}
// SchemaVersion reads the applied goose version without migrating. An
// unmigrated database is version 0.
func SchemaVersion(ctx context.Context, pool *pgxpool.Pool) (int64, error) {
var version int64
err := pool.QueryRow(ctx, `SELECT CASE WHEN to_regclass('public.goose_db_version') IS NULL THEN 0
ELSE (SELECT coalesce(max(version_id), 0) FROM public.goose_db_version WHERE is_applied) END`).Scan(&version)
if err != nil {
return 0, fmt.Errorf("postgres: read schema version: %w", err)
}
return version, nil
}
// AcquireAll claims every name, or none: on any failure the claims already
// taken are released before the error returns.
func AcquireAll(ctx context.Context, pool *pgxpool.Pool, names []string) (release func(context.Context), err error) {
held := make([]*Ownership, 0, len(names))
release = func(ctx context.Context) {
for _, o := range slices.Backward(held) {
_ = o.Release(ctx)
}
}
for _, name := range names {
o, err := AcquireOwnership(ctx, pool, name)
if err != nil {
release(ctx)
return nil, err
}
held = append(held, o)
}
return release, nil
}
// Backup writes a whole-database pg_dump archive to out. It takes no claim:
// pg_dump reads one consistent snapshot, so it is safe against serving
// processes, which is what the update installer needs. The archive goes to a
// scratch file and replaces out only once pg_restore can read it back, so a
// failed run never costs the previous good snapshot.
func Backup(ctx context.Context, databaseURL, out string) error {
tmp, err := os.CreateTemp(filepath.Dir(out), filepath.Base(out)+".tmp*")
if err != nil {
return fmt.Errorf("postgres: backup: %w", err)
}
scratch := tmp.Name()
_ = tmp.Close()
defer func() { _ = os.Remove(scratch) }()
dbname, env := libpqTarget(databaseURL)
if _, err := runTool(ctx, env, "pg_dump", dumpArgs(dbname, scratch)...); err != nil {
return fmt.Errorf("postgres: backup: %w", err)
}
if _, err := ValidateSnapshot(ctx, scratch); err != nil {
return fmt.Errorf("postgres: backup produced an unreadable snapshot: %w", err)
}
if err := os.Rename(scratch, out); err != nil {
return fmt.Errorf("postgres: backup: %w", err)
}
return nil
}
// ValidateSnapshot reads the archive's table of contents and returns the
// tables it holds. An archive without goose's version table is not a dump of
// an archie database.
func ValidateSnapshot(ctx context.Context, path string) ([]string, error) {
if _, err := os.Stat(path); err != nil {
return nil, fmt.Errorf("postgres: snapshot: %w", err)
}
list, err := runTool(ctx, nil, "pg_restore", "--list", path)
if err != nil {
return nil, fmt.Errorf("postgres: snapshot %s: %w", path, err)
}
tables := snapshotTables(list)
if !slices.Contains(tables, gooseTable) {
return nil, fmt.Errorf("postgres: snapshot %s holds no %s; it is not an archie database dump", path, gooseTable)
}
return tables, nil
}
// Restore replaces the whole database with the snapshot at from. It is
// destructive: every write after the snapshot is lost. It refuses unless it
// can claim every service role, so no serving process is running and none can
// start until it finishes; a claim that cannot be checked is a refusal too.
//
// pg_restore --clean drops only what the archive holds, so tables a later
// migration created are dropped first; otherwise they would survive into the
// restored schema and break the next upgrade's migration.
func Restore(ctx context.Context, databaseURL, from string) error {
tables, err := ValidateSnapshot(ctx, from)
if err != nil {
return err
}
pool, err := Open(ctx, databaseURL)
if err != nil {
return fmt.Errorf("postgres: restore cannot confirm every service is stopped: %w", err)
}
defer pool.Close()
release, err := AcquireAll(ctx, pool, ServiceOwners)
if err != nil {
return fmt.Errorf("postgres: restore refused, stop every archie service first: %w", err)
}
defer release(context.WithoutCancel(ctx))
if err := dropTablesNotIn(ctx, pool, tables); err != nil {
return err
}
dbname, env := libpqTarget(databaseURL)
if _, err := runTool(ctx, env, "pg_restore", restoreArgs(dbname, from)...); err != nil {
return fmt.Errorf("postgres: restore: %w", err)
}
return nil
}
func dropTablesNotIn(ctx context.Context, pool *pgxpool.Pool, keep []string) error {
rows, err := pool.Query(ctx, `SELECT schemaname, tablename FROM pg_tables
WHERE schemaname NOT IN ('pg_catalog', 'information_schema')`)
if err != nil {
return fmt.Errorf("postgres: restore: list tables: %w", err)
}
var drop []pgx.Identifier
for rows.Next() {
var schema, table string
if err := rows.Scan(&schema, &table); err != nil {
rows.Close()
return fmt.Errorf("postgres: restore: list tables: %w", err)
}
if !slices.Contains(keep, schema+"."+table) {
drop = append(drop, pgx.Identifier{schema, table})
}
}
rows.Close()
if err := rows.Err(); err != nil {
return fmt.Errorf("postgres: restore: list tables: %w", err)
}
for _, table := range drop {
if _, err := pool.Exec(ctx, "DROP TABLE IF EXISTS "+table.Sanitize()+" CASCADE"); err != nil {
return fmt.Errorf("postgres: restore: drop %s: %w", table.Sanitize(), err)
}
}
return nil
}
// snapshotTables extracts schema.table for every TABLE entry of a
// `pg_restore --list` listing, sorted.
func snapshotTables(list string) []string {
var tables []string
for line := range strings.Lines(list) {
if strings.HasPrefix(line, ";") {
continue
}
_, entry, ok := strings.Cut(line, ";")
if !ok {
continue
}
fields := strings.Fields(entry)
// "tableoid oid TABLE schema name owner"; "TABLE DATA" entries are rows.
if len(fields) >= 5 && fields[2] == "TABLE" && fields[3] != "DATA" {
tables = append(tables, fields[3]+"."+fields[4])
}
}
slices.Sort(tables)
return tables
}
func dumpArgs(dbname, out string) []string {
return []string{"--format=custom", "--no-password", "--file=" + out, "--dbname=" + dbname}
}
func restoreArgs(dbname, from string) []string {
return []string{"--clean", "--if-exists", "--single-transaction", "--exit-on-error", "--no-owner", "--no-password", "--dbname=" + dbname, from}
}
// libpqTarget splits a connection URL into the dbname argument and the
// environment the client tools read, moving the password to PGPASSWORD so it
// never appears in a process listing. A key=value connection string is passed
// through as-is.
func libpqTarget(databaseURL string) (string, []string) {
u, err := url.Parse(databaseURL)
if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") || u.User == nil {
return databaseURL, nil
}
password, ok := u.User.Password()
if !ok {
return databaseURL, nil
}
u.User = url.User(u.User.Username())
return u.String(), []string{"PGPASSWORD=" + password}
}
// runTool runs a PostgreSQL client tool and returns its stdout; a failure
// carries its stderr, which is where the tools explain themselves.
func runTool(ctx context.Context, env []string, name string, args ...string) (string, error) {
cmd := exec.CommandContext(ctx, name, args...)
cmd.Env = append(os.Environ(), env...)
var stdout, stderr bytes.Buffer
cmd.Stdout, cmd.Stderr = &stdout, &stderr
if err := cmd.Run(); err != nil {
return "", fmt.Errorf("%s: %w: %s", name, err, strings.TrimSpace(stderr.String()))
}
return stdout.String(), nil
}
@@ -0,0 +1,302 @@
package postgres
import (
"errors"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
)
func TestLibpqTargetKeepsThePasswordOffTheCommandLine(t *testing.T) {
tests := []struct {
name string
url string
wantDB string
wantEnv []string
leakFree string
}{
{
name: "url with password",
url: "postgres://archie:s3cret@db.example:5432/archie?sslmode=require",
wantDB: "postgres://archie@db.example:5432/archie?sslmode=require",
wantEnv: []string{"PGPASSWORD=s3cret"},
leakFree: "s3cret",
},
{
name: "url without password",
url: "postgresql://archie@db.example/archie",
wantDB: "postgresql://archie@db.example/archie",
},
{
name: "key value connection string passes through",
url: "host=db.example dbname=archie",
wantDB: "host=db.example dbname=archie",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db, env := libpqTarget(tt.url)
if db != tt.wantDB {
t.Errorf("dbname = %q, want %q", db, tt.wantDB)
}
if !slices.Equal(env, tt.wantEnv) {
t.Errorf("env = %q, want %q", env, tt.wantEnv)
}
if tt.leakFree != "" && strings.Contains(db, tt.leakFree) {
t.Errorf("dbname %q carries the password", db)
}
})
}
}
func TestToolArguments(t *testing.T) {
tests := []struct {
name string
got []string
want []string
}{
{
name: "dump is a whole-database custom-format archive",
got: dumpArgs("postgres://h/db", "/tmp/out.dump"),
want: []string{"--format=custom", "--no-password", "--file=/tmp/out.dump", "--dbname=postgres://h/db"},
},
{
name: "restore cleans, stops on the first error and is atomic",
got: restoreArgs("postgres://h/db", "/tmp/in.dump"),
want: []string{"--clean", "--if-exists", "--single-transaction", "--exit-on-error", "--no-owner", "--no-password", "--dbname=postgres://h/db", "/tmp/in.dump"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if !slices.Equal(tt.got, tt.want) {
t.Errorf("args = %q, want %q", tt.got, tt.want)
}
})
}
}
func TestSnapshotTablesReadsTheArchiveList(t *testing.T) {
list := `;
; Archive created at 2026-09-24 10:00:00 UTC
;
215; 1259 16390 TABLE public tasks archie
216; 1259 16391 SEQUENCE public tasks_id_seq archie
3401; 0 16390 TABLE DATA public tasks archie
217; 1259 16400 TABLE public goose_db_version archie
`
got := snapshotTables(list)
want := []string{"public.goose_db_version", "public.tasks"}
if !slices.Equal(got, want) {
t.Fatalf("snapshotTables = %q, want %q", got, want)
}
}
func requireTools(t *testing.T) {
t.Helper()
for _, tool := range []string{"pg_dump", "pg_restore"} {
if _, err := exec.LookPath(tool); err != nil {
t.Skipf("%s is not on PATH; the integration test needs the PostgreSQL 18 client tools", tool)
}
}
}
// migratedURL is a migrated database and a pool on it.
func migratedURL(t *testing.T) (string, *pgxpool.Pool) {
t.Helper()
url := pgtest.URL(t)
pool := openPool(t, url)
if err := Migrate(t.Context(), pool, Migrations()); err != nil {
t.Fatalf("Migrate: %v", err)
}
return url, pool
}
func titles(t *testing.T, pool *pgxpool.Pool) []string {
t.Helper()
rows, err := pool.Query(t.Context(), "SELECT title FROM tasks ORDER BY id")
if err != nil {
t.Fatalf("list tasks: %v", err)
}
defer rows.Close()
var out []string
for rows.Next() {
var title string
if err := rows.Scan(&title); err != nil {
t.Fatal(err)
}
out = append(out, title)
}
return out
}
func enqueue(t *testing.T, pool *pgxpool.Pool, title string) {
t.Helper()
if _, err := New(pool).EnqueueChatTask(t.Context(), "acme", "widget", title, "body", "implement", ""); err != nil {
t.Fatalf("enqueue %q: %v", title, err)
}
}
// A restore after a forward migration is the case restore exists for: the
// tables the later migration added are not in the snapshot, so --clean alone
// would leave them behind for the next upgrade to trip over.
func TestBackupThenRestoreReturnsTheDatabaseToTheSnapshot(t *testing.T) {
requireTools(t)
url, pool := migratedURL(t)
enqueue(t, pool, "before")
before, err := SchemaVersion(t.Context(), pool)
if err != nil {
t.Fatalf("SchemaVersion: %v", err)
}
snapshot := filepath.Join(t.TempDir(), "snapshot.dump")
if err := Backup(t.Context(), url, snapshot); err != nil {
t.Fatalf("Backup: %v", err)
}
if leftovers, _ := filepath.Glob(snapshot + ".tmp*"); len(leftovers) != 0 {
t.Errorf("backup left scratch files: %v", leftovers)
}
enqueue(t, pool, "after")
if _, err := pool.Exec(t.Context(), "CREATE TABLE later_migration (id bigint)"); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(t.Context(), "INSERT INTO goose_db_version (version_id, is_applied) VALUES ($1, true)", before+1); err != nil {
t.Fatal(err)
}
if err := Restore(t.Context(), url, snapshot); err != nil {
t.Fatalf("Restore: %v", err)
}
if got := titles(t, pool); !slices.Equal(got, []string{"before"}) {
t.Errorf("tasks after restore = %q, want only the snapshot's", got)
}
var extra bool
if err := pool.QueryRow(t.Context(), "SELECT to_regclass('public.later_migration') IS NOT NULL").Scan(&extra); err != nil {
t.Fatal(err)
}
if extra {
t.Error("restore left a table the snapshot does not contain")
}
if after, err := SchemaVersion(t.Context(), pool); err != nil || after != before {
t.Errorf("schema version after restore = %d, %v; want %d", after, err, before)
}
enqueue(t, pool, "writable")
}
func TestRestoreRefusesWhileAnyServiceIsServing(t *testing.T) {
requireTools(t)
for _, owner := range ServiceOwners {
t.Run(owner, func(t *testing.T) {
url, pool := migratedURL(t)
enqueue(t, pool, "snapshot")
snapshot := filepath.Join(t.TempDir(), "snapshot.dump")
if err := Backup(t.Context(), url, snapshot); err != nil {
t.Fatalf("Backup: %v", err)
}
enqueue(t, pool, "live")
claimOther(t, url, owner, false, false)
err := Restore(t.Context(), url, snapshot)
if !errors.Is(err, ErrOwned) {
t.Fatalf("Restore with %s serving = %v, want ErrOwned", owner, err)
}
if !strings.Contains(err.Error(), owner) {
t.Errorf("refusal must name the serving role %q: %v", owner, err)
}
if got := titles(t, pool); !slices.Equal(got, []string{"snapshot", "live"}) {
t.Errorf("refused restore changed the database: %q", got)
}
// Every claim the refused restore took is dropped again.
for _, other := range ServiceOwners {
if other == owner {
continue
}
claim, err := AcquireOwnership(t.Context(), pool, other)
if err != nil {
t.Fatalf("%s still held after a refused restore: %v", other, err)
}
_ = claim.Release(t.Context())
}
})
}
}
func TestRestoreRefusesWhatItCannotCheck(t *testing.T) {
requireTools(t)
url, pool := migratedURL(t)
snapshot := filepath.Join(t.TempDir(), "snapshot.dump")
if err := Backup(t.Context(), url, snapshot); err != nil {
t.Fatalf("Backup: %v", err)
}
notASnapshot := filepath.Join(t.TempDir(), "garbage.dump")
if err := os.WriteFile(notASnapshot, []byte("not an archive"), 0o600); err != nil {
t.Fatal(err)
}
enqueue(t, pool, "live")
tests := []struct {
name string
url string
from string
}{
{name: "unreachable server", url: "postgres://archie:archie@127.0.0.1:1/none?connect_timeout=2", from: snapshot},
{name: "not a snapshot", url: url, from: notASnapshot},
{name: "missing snapshot", url: url, from: filepath.Join(t.TempDir(), "absent.dump")},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := Restore(t.Context(), tt.url, tt.from); err == nil {
t.Fatal("Restore succeeded, want refusal")
}
if got := titles(t, pool); !slices.Equal(got, []string{"live"}) {
t.Errorf("refused restore changed the database: %q", got)
}
})
}
}
func TestBackupFailureKeepsThePreviousSnapshot(t *testing.T) {
requireTools(t)
out := filepath.Join(t.TempDir(), "snapshot.dump")
if err := os.WriteFile(out, []byte("good snapshot"), 0o600); err != nil {
t.Fatal(err)
}
if err := Backup(t.Context(), "postgres://archie:archie@127.0.0.1:1/none?connect_timeout=2", out); err == nil {
t.Fatal("Backup of an unreachable database succeeded")
}
if data, _ := os.ReadFile(out); string(data) != "good snapshot" {
t.Errorf("failed backup replaced the previous snapshot: %q", data)
}
if leftovers, _ := filepath.Glob(out + ".tmp*"); len(leftovers) != 0 {
t.Errorf("failed backup left scratch files: %v", leftovers)
}
}
func TestCheckSchemaVersion(t *testing.T) {
supported := SupportedSchemaVersion()
if supported < 4 {
t.Fatalf("SupportedSchemaVersion = %d, want the highest embedded migration", supported)
}
tests := []struct {
name string
version int64
wantErr error
}{
{name: "current", version: supported},
{name: "older is upgradable", version: 1},
{name: "newer than this binary", version: supported + 1, wantErr: ErrSchemaTooNew},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := CheckSchemaVersion(tt.version); !errors.Is(err, tt.wantErr) {
t.Fatalf("CheckSchemaVersion(%d) = %v, want %v", tt.version, err, tt.wantErr)
}
})
}
}
@@ -0,0 +1,216 @@
package postgres
import (
"context"
"strings"
"testing"
"unicode/utf8"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
)
func storeFor(t *testing.T) *Store {
t.Helper()
pool := openPool(t, pgtest.URL(t))
if err := Migrate(t.Context(), pool, Migrations()); err != nil {
t.Fatalf("Migrate: %v", err)
}
return New(pool)
}
// runningTask enqueues one issue and claims it, the state every park starts in.
func runningTask(t *testing.T, s *Store) *workflow.Task {
t.Helper()
if _, err := s.EnqueueIssue(t.Context(), "acme", "widgets", 1, "t", "b", "", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(t.Context())
if err != nil || task == nil {
t.Fatalf("ClaimNext = %v, %v", task, err)
}
return task
}
type transitionRow struct{ from, to, detail string }
func transitions(t *testing.T, s *Store, taskID int64) []transitionRow {
t.Helper()
rows, err := s.pool.Query(t.Context(), "SELECT from_status, to_status, detail FROM transitions WHERE task_id = $1 ORDER BY id", taskID)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var out []transitionRow
for rows.Next() {
var r transitionRow
if err := rows.Scan(&r.from, &r.to, &r.detail); err != nil {
t.Fatal(err)
}
out = append(out, r)
}
return out
}
// Every free-text column the SQLite store clipped to 4000 bytes on a rune
// boundary is clipped the same way here. The oversized value ends in a
// multi-byte rune straddling the limit, so a byte cut would store invalid
// UTF-8 and a missing clip would store all of it.
func TestWritesClipFreeTextTo4000Bytes(t *testing.T) {
oversized := strings.Repeat("x", 3999) + "é" + strings.Repeat("y", 1000)
want := strings.Repeat("x", 3999)
tests := []struct {
name string
write func(t *testing.T, s *Store) string
}{
{name: "transition to parked: park reason and transition detail", write: func(t *testing.T, s *Store) string {
task := runningTask(t, s)
if err := s.Transition(t.Context(), task.ID, workflow.StatusRunning, workflow.StatusParked, oversized); err != nil {
t.Fatal(err)
}
rows := transitions(t, s, task.ID)
if len(rows) == 0 || rows[len(rows)-1].detail != want {
t.Errorf("transition detail not clipped: %d rows", len(rows))
}
got, _ := s.TaskByID(t.Context(), task.ID)
return got.ParkReason
}},
{name: "park task: park reason and transition detail", write: func(t *testing.T, s *Store) string {
task := runningTask(t, s)
if err := s.ParkTask(t.Context(), task.ID, workflow.StatusRunning, oversized, "transient"); err != nil {
t.Fatal(err)
}
rows := transitions(t, s, task.ID)
if len(rows) == 0 || rows[len(rows)-1].detail != want {
t.Errorf("transition detail not clipped: %d rows", len(rows))
}
got, _ := s.TaskByID(t.Context(), task.ID)
return got.ParkReason
}},
{name: "update: park reason", write: func(t *testing.T, s *Store) string {
task := runningTask(t, s)
task.ParkReason = oversized
if err := s.Update(t.Context(), task); err != nil {
t.Fatal(err)
}
got, _ := s.TaskByID(t.Context(), task.ID)
return got.ParkReason
}},
{name: "begin remediation: review payload", write: func(t *testing.T, s *Store) string {
task := runningTask(t, s)
if err := s.Transition(t.Context(), task.ID, workflow.StatusRunning, workflow.StatusPROpen, "pr"); err != nil {
t.Fatal(err)
}
if err := s.BeginRemediation(t.Context(), task.ID, oversized); err != nil {
t.Fatal(err)
}
got, _ := s.TaskByID(t.Context(), task.ID)
return got.ReviewPayload
}},
{name: "update review payload", write: func(t *testing.T, s *Store) string {
task := runningTask(t, s)
if err := s.Transition(t.Context(), task.ID, workflow.StatusRunning, workflow.StatusPROpen, "pr"); err != nil {
t.Fatal(err)
}
if err := s.BeginRemediation(t.Context(), task.ID, "first"); err != nil {
t.Fatal(err)
}
if err := s.UpdateReviewPayload(t.Context(), task.ID, oversized); err != nil {
t.Fatal(err)
}
got, _ := s.TaskByID(t.Context(), task.ID)
return got.ReviewPayload
}},
{name: "event detail", write: func(t *testing.T, s *Store) string {
if _, err := s.InsertEvent(t.Context(), events.Event{Kind: events.KindLog, Detail: oversized}); err != nil {
t.Fatal(err)
}
got, err := s.EventsSince(t.Context(), "", 1)
if err != nil || len(got) != 1 {
t.Fatalf("EventsSince = %d, %v", len(got), err)
}
return got[0].Detail
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.write(t, storeFor(t))
if got != want || !utf8.ValidString(got) {
t.Errorf("stored %d bytes (valid UTF-8 %v), want the %d-byte rune-boundary clip", len(got), utf8.ValidString(got), len(want))
}
})
}
}
func TestParkClassNormalisation(t *testing.T) {
tests := []struct {
name string
park func(ctx context.Context, s *Store, id int64) error
want string
}{
{name: "generic transition to parked is needs_human", want: "needs_human", park: func(ctx context.Context, s *Store, id int64) error {
return s.Transition(ctx, id, workflow.StatusRunning, workflow.StatusParked, "gate failed")
}},
{name: "known class is kept", want: "transient", park: func(ctx context.Context, s *Store, id int64) error {
return s.ParkTask(ctx, id, workflow.StatusRunning, "container acquire failed", "transient")
}},
{name: "unknown class falls back to needs_human", want: "needs_human", park: func(ctx context.Context, s *Store, id int64) error {
return s.ParkTask(ctx, id, workflow.StatusRunning, "x", "urgent")
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := storeFor(t)
task := runningTask(t, s)
if err := tt.park(t.Context(), s, task.ID); err != nil {
t.Fatal(err)
}
got, _ := s.TaskByID(t.Context(), task.ID)
if got.Status != workflow.StatusParked || got.ParkClass != tt.want {
t.Fatalf("status %q class %q, want parked %q", got.Status, got.ParkClass, tt.want)
}
// Requeue clears class and reason together: a stale class on a
// queued task would group a live task with parked ones.
if err := s.Requeue(t.Context(), task.ID, workflow.StatusParked, "implement"); err != nil {
t.Fatal(err)
}
got, _ = s.TaskByID(t.Context(), task.ID)
if got.ParkClass != "needs_human" || got.ParkReason != "" {
t.Errorf("after requeue: class %q reason %q, want cleared", got.ParkClass, got.ParkReason)
}
})
}
}
// A transition that leaves parked alone keeps the previous park fields, and
// every guarded write leaves exactly one audit row in order.
func TestTransitionsRecordEachMoveInOrder(t *testing.T) {
s := storeFor(t)
task := runningTask(t, s)
ctx := t.Context()
if err := s.ParkTask(ctx, task.ID, workflow.StatusRunning, "why", "transient"); err != nil {
t.Fatal(err)
}
if err := s.Requeue(ctx, task.ID, workflow.StatusParked, "implement"); err != nil {
t.Fatal(err)
}
// A stale transition writes no audit row.
if err := s.Transition(ctx, task.ID, workflow.StatusRunning, workflow.StatusPROpen, "stale"); err == nil {
t.Fatal("stale Transition succeeded")
}
got := transitions(t, s, task.ID)
want := []transitionRow{
{workflow.StatusRunning, workflow.StatusParked, "why"},
{workflow.StatusParked, workflow.StatusQueued, "requeued implement"},
}
if len(got) != len(want) {
t.Fatalf("transitions = %+v, want %+v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Errorf("transition %d = %+v, want %+v", i, got[i], want[i])
}
}
}
@@ -8,6 +8,12 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
)
// The serving roles that each hold an ownership claim for their whole life.
const (
OwnerStateStore = "state-store"
OwnerGateway = "gateway"
)
// ErrOwned is returned when another process already owns the named role on
// this database.
var ErrOwned = errors.New("postgres: owned by another process")
@@ -0,0 +1,114 @@
package releaseupdate
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
const snapshotConfig = "database_url = 'postgres://archie:hunter2@db.example/archie'\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n"
// The new State Store migrates on start and nothing undoes a migration
// automatically, so a daemon update snapshots the whole database first,
// through the release's own recovery subcommand, and hands the snapshot's
// path to the watchdog. The connection string stays inside the config.
func TestUpdateInstallSnapshotsTheDatabaseBeforeMigrating(t *testing.T) {
snapshot := filepath.Join(t.TempDir(), "pre-update.dump")
_, calls, _ := runUpdateInstallWithConfig(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_DB_SNAPSHOT": snapshot,
}, snapshotConfig, false)
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-config ", "-out "+snapshot)
assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_DB_SNAPSHOT="+snapshot)
assertCallAbsent(t, calls, "hunter2")
}
func TestUpdateInstallSnapshotScope(t *testing.T) {
tests := []struct {
name string
env map[string]string
wantSnapshot bool
wantRefusal bool
}{
{name: "agent-only update does not touch the database", env: map[string]string{
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", "ARCHIE_UPDATE_AGENT_VERSION": "1.22.0",
}},
{name: "daemon update refuses when the snapshot fails", wantSnapshot: true, wantRefusal: true, env: map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0", "ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", "ARCHIE_TEST_GO_FAILS_SNAPSHOT": "1",
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, calls, output := runUpdateInstallWithConfig(t, tt.env, snapshotConfig, tt.wantRefusal)
snapshotted := false
for _, call := range calls {
if strings.Contains(call, "archie-state-store backup") && strings.Contains(call, "-config ") {
snapshotted = true
}
}
if snapshotted != tt.wantSnapshot {
t.Fatalf("snapshot attempted = %v, want %v; calls = %q", snapshotted, tt.wantSnapshot, calls)
}
if !tt.wantRefusal {
return
}
if !strings.Contains(output, "could not snapshot the database") {
t.Errorf("refusal must say the snapshot failed; output = %q", output)
}
for _, call := range calls {
if strings.Contains(call, "install -m755") && !strings.Contains(call, "/scripts/") {
t.Errorf("refused update still installed %q", call)
}
}
})
}
}
// A failed daemon update rolls the binaries back but never the database, and
// says so, naming the explicit offline restore rather than running one.
func TestUpdateWatchdogNeverRestoresTheDatabaseItself(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
for name, content := range map[string]string{"archied": "new", "archied.prev": "old"} {
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
callsPath := filepath.Join(work, "calls")
writeFakeCommand(t, fakeDir, "systemctl", `exit 0`)
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
for _, tool := range []string{"pg_restore", "psql", "go", "archie-state-store"} {
writeFakeCommand(t, fakeDir, tool, `printf '%s\n' "`+tool+` $*" >> "$ARCHIE_TEST_CALLS"`)
}
snapshot := filepath.Join(work, "pre-update.dump")
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
"ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_REPORT_PATH="+filepath.Join(work, "report.json"),
"ARCHIE_UPDATE_HEALTH_TIMEOUT=0", "ARCHIE_UPDATE_COMPONENTS=daemon",
"ARCHIE_DB_SNAPSHOT="+snapshot, "ARCHIE_CONFIG_PATH=/etc/archie/config.toml")
output, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
if calls, _ := os.ReadFile(callsPath); len(calls) != 0 {
t.Errorf("watchdog ran database tooling itself: %q", calls)
}
for _, want := range []string{"NOT rolled back", "archie-state-store restore -config /etc/archie/config.toml -from " + snapshot, "lost"} {
if !strings.Contains(string(output), want) {
t.Errorf("output lacks %q:\n%s", want, output)
}
}
}
@@ -411,6 +411,8 @@ printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
# must refuse the update.
if [ "$1" = "run" ]; then
[ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1
# The PostgreSQL snapshot is the backup run that names no -db file.
[ "${ARCHIE_TEST_GO_FAILS_SNAPSHOT:-}" = "1" ] && [[ " $* " != *" -db "* ]] && exit 1
exit 0
fi
out=""
+19
View File
@@ -80,6 +80,7 @@ AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}"
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}"
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
# The address the daemon dials for its state, so the watchdog can wait for it to
# accept connections before it starts archied. Read with the same narrow awk as
# the task database path and for the same reason: the daemon stays the authority
@@ -284,6 +285,22 @@ if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_
fi
fi
# The pre-migration snapshot of the PostgreSQL database: the whole database,
# since every service shares it. The new State Store migrates on start and a
# migration is never undone automatically (goose Down drops tables), so this
# snapshot is the only way back, restored by an operator with every service
# stopped. The recovery subcommand resolves database_url from the same config
# the services boot with, so the connection string never passes through this
# script. No snapshot, no update.
if [[ ",$components," == *,daemon,* ]]; then
echo "==> snapshotting the database before the update migrates it"
mkdir -p "$(dirname "$DB_SNAPSHOT")"
if ! go run ./cmd/archie-state-store backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT"; then
echo "refusing to install $GW: could not snapshot the database named by $CONFIG_PATH, so a failed migration could not be recovered" >&2
exit 1
fi
fi
# Refresh adapters from the approved release source. The running shell
# continues with this copy; subsequent updates use the current contract.
install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install"
@@ -377,6 +394,8 @@ systemd-run --user --unit=archied-update-restart --collect --quiet \
--setenv=ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS="${AGENT_VERSION_PREVIOUS_EXISTS:-0}" \
--setenv=ARCHIE_TASK_DB_PATH="$TASK_DB_PATH" \
--setenv=ARCHIE_TASK_DB_BACKUP="$TASK_DB_BACKUP" \
--setenv=ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT" \
--setenv=ARCHIE_CONFIG_PATH="$CONFIG_PATH" \
--setenv=ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET" \
"$BIN_DIR/archie-update-watchdog"
+15
View File
@@ -15,6 +15,8 @@ AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.versi
AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}"
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}"
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}"
# The daemon exports its own health address (releaseupdate.CommandInstaller);
# this default only covers a hand-driven run, and must stay the address archied
# itself serves /healthz on -- [health] listen, defaulted in
@@ -218,6 +220,16 @@ rollback_task_db() {
rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm"
}
# explain_database_recovery states what a daemon rollback does not do. The
# PostgreSQL database is never restored automatically: goose Down drops tables,
# and a restore is destructive, loses every write after the snapshot and needs
# every service stopped, so it is an operator's decision, not this script's.
explain_database_recovery() {
echo "==> the PostgreSQL database was NOT rolled back; schema migrations are never undone automatically" >&2
echo "==> if the previous release refuses the migrated schema, stop every archie service and restore the pre-update snapshot (every write made after it is lost):" >&2
echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2
}
start_units
if wait_healthy; then
@@ -273,5 +285,8 @@ fi
if [ "$rolled_back" = true ]; then
start_units
fi
if [[ ",$COMPONENTS," == *,daemon,* ]]; then
explain_database_recovery
fi
write_report "failed" "$rolled_back"