feat(archied): recovery commands on PostgreSQL; fail-closed pre-update snapshot, no automatic schema rollback
Without -db, backup/restore/validate/rollback resolve database_url from the config and work on the whole PostgreSQL database. Restore claims every service role; rollback claims the State Store role; validate refuses a schema newer than the binary. The installer refuses a daemon update it cannot snapshot first; the watchdog never restores the database and prints the explicit offline restore instead.
This commit is contained in:
1 parent
b39f7c682f
commit
fe819d489a
11 files changed
+588
-48
No files matched your search
@@ -19,18 +19,28 @@ const recoveryUsage = `usage: archie-state-store <command> [flags]
|
||||
|
||||
With no command, archie-state-store serves the State Store gRPC contract.
|
||||
|
||||
The recovery commands operate on the task database file directly -- the
|
||||
configured db_path with "-tasks.sqlite" appended -- and are how an operator
|
||||
recovers a database the control plane will not start on. Stop the State Store
|
||||
first: restore and rollback rewrite the file and refuse while another process
|
||||
owns it, while backup takes a consistent snapshot of a serving store.
|
||||
The recovery commands are how an operator recovers a database the control
|
||||
plane will not start on. Without -db they operate on the PostgreSQL database
|
||||
the configuration's database_url names, through pg_dump and pg_restore (which
|
||||
must be on PATH); with -db, on a legacy SQLite task file (the configured
|
||||
db_path with "-tasks.sqlite" appended).
|
||||
|
||||
backup -db FILE -out FILE write a snapshot of the database
|
||||
restore -db FILE -from FILE replace the database with a snapshot
|
||||
validate -db FILE [-config FILE] check the file, its stored settings and the
|
||||
configuration the daemon would boot with
|
||||
rollback -db FILE -kind KIND replay an earlier revision of a stored
|
||||
backup takes a consistent snapshot of a serving database. restore is
|
||||
destructive and offline: stop every archie service first, since it refuses
|
||||
while any of them holds its ownership claim, and every write made after the
|
||||
snapshot is lost. rollback refuses while the State Store serves. There is no
|
||||
automatic schema rollback: the way back from a migration is restoring the
|
||||
snapshot taken before it.
|
||||
|
||||
backup [-db FILE] -out FILE write a snapshot of the database
|
||||
restore [-db FILE] -from FILE replace the database with a snapshot
|
||||
validate [-db FILE] check the database, its stored settings and
|
||||
the configuration the daemon would boot with
|
||||
rollback [-db FILE] -kind KIND replay an earlier revision of a stored
|
||||
[-revision N] resource through the ordinary replace
|
||||
|
||||
Every command takes -config FILE and -config-overlay FILE: the configuration
|
||||
the daemon boots with, which names the database.
|
||||
`
|
||||
|
||||
// runRecovery parses and performs one offline recovery command, reporting the
|
||||
@@ -54,17 +64,15 @@ func runRecovery(args []string, stdout, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("archie-state-store "+command, flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
flags.Usage = func() { fmt.Fprint(stderr, recoveryUsage) }
|
||||
flags.StringVar(&options.DB, "db", "", "task database file the State Store owns (<db_path>-tasks.sqlite)")
|
||||
flags.StringVar(&options.DB, "db", "", "legacy SQLite task file (<db_path>-tasks.sqlite); omit to use the configured database_url")
|
||||
flags.StringVar(&options.Out, "out", "", "snapshot file backup writes")
|
||||
flags.StringVar(&options.From, "from", "", "snapshot file restore reads")
|
||||
flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays")
|
||||
flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)")
|
||||
// Only validate asks a question about the process rather than the file, so
|
||||
// only validate takes the configuration the process would boot with.
|
||||
if command == archied.RecoveryValidate {
|
||||
flags.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory the daemon boots with")
|
||||
flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
|
||||
}
|
||||
// The configuration names the PostgreSQL database, and validate also checks
|
||||
// the stored settings against it.
|
||||
flags.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory the daemon boots with")
|
||||
flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
|
||||
if err := flags.Parse(args[1:]); err != nil {
|
||||
// A requested help is not a wrong command line: the usage above is the
|
||||
// answer, and the serve path exits 0 for the same request.
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Without -db every recovery command targets the configured PostgreSQL
|
||||
// database, so each one accepts -config and refuses, naming the key, when the
|
||||
// configuration names no database.
|
||||
func TestRecoveryWithoutDBUsesTheConfiguredDatabase(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath := writeMinimalConfig(t, dir)
|
||||
tests := [][]string{
|
||||
{"backup", "-out", filepath.Join(dir, "snapshot.dump")},
|
||||
{"restore", "-from", filepath.Join(dir, "snapshot.dump")},
|
||||
{"validate"},
|
||||
{"rollback", "-kind", "settings"},
|
||||
}
|
||||
for _, args := range tests {
|
||||
t.Run(args[0], func(t *testing.T) {
|
||||
code, _, stderr := runRecoveryCmd(t, append(args, "-config", configPath)...)
|
||||
if code != 1 || !strings.Contains(stderr, "database_url") {
|
||||
t.Fatalf("%s exited %d, stderr %q; want 1 naming database_url", args[0], code, stderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -165,6 +165,31 @@ own `sqlite3` shell, so the installer and an operator run one implementation.
|
||||
blunt one, and is also the way back from a schema migration a release cannot
|
||||
grow out of.
|
||||
|
||||
### The same commands on PostgreSQL
|
||||
|
||||
Without `-db`, each command works on the PostgreSQL database the configuration's
|
||||
`database_url` names (`internal/app/archied/state_store_recovery_postgres.go`,
|
||||
`internal/infrastructure/postgres/maintenance.go`). One database backs every
|
||||
service, so the scope is always the whole database.
|
||||
|
||||
- `backup -out F` runs `pg_dump --format=custom`, reads the archive back with
|
||||
`pg_restore --list` and only then replaces `F`. It takes no claim and runs
|
||||
against a serving database.
|
||||
- `restore -from F` is destructive and offline. It claims every service role's
|
||||
ownership lock (`state-store` and `gateway`) without waiting and refuses if
|
||||
any is held or the check cannot complete, then drops tables the snapshot does
|
||||
not hold and runs `pg_restore --clean --single-transaction`. Every write made
|
||||
after the snapshot is lost.
|
||||
- `validate` refuses a schema newer than the binary's highest migration, then
|
||||
runs the same two validation layers.
|
||||
- `rollback` holds the `state-store` claim for the replay.
|
||||
|
||||
There is no automatic schema rollback. goose `Down` drops tables, so it is data
|
||||
loss, not a rollback. `scripts/archie-update-install` refuses a daemon update
|
||||
it cannot snapshot first, and on a failed update `scripts/archie-update-watchdog`
|
||||
rolls the binaries back, leaves the database alone and prints the `restore`
|
||||
command for the pre-update snapshot. Whether to run it is the operator's call.
|
||||
|
||||
## Gateway restart: two constraints learned the hard way
|
||||
|
||||
Telegram `/restart` (`internal/channels/telegram/restart.go` plus the `Start`
|
||||
|
||||
@@ -100,7 +100,7 @@
|
||||
"url": "/dev/architecture/safe-change-and-recovery/",
|
||||
"title": "Safe Change and Recovery",
|
||||
"section": "Architecture",
|
||||
"body": "# Safe Change and Recovery\n\n**Status:** Requirements approved; mechanics are under design. Database-backed settings are implemented through the control plane (2026-09): see the degrade/recovery section below, which records a gap rather than a solution. The full runtime-supervision protocol (bounded observation, versioned audit trail) remains under design.\n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Purpose\n\nArchie is built and improved by agents as well as humans. The architecture MUST\nmake a change easy to locate, understand, test, audit, extend, and reverse\nwithout requiring its author to understand unrelated parts of the application.\n\nSeparation of concerns is therefore an operational safety boundary, not only a\ncode-organisation preference.\n\n## Required outcomes\n\n- An agent can improve a focused feature without receiving authority over\n unrelated domains.\n- Every event, function, action, setting, and policy has an identifiable owner\n and provenance.\n- Source and runtime changes pass deterministic checks outside the actor making\n the change.\n- A rejected or unhealthy candidate cannot replace the last known working\n version.\n- Invalid user configuration does not terminate an otherwise healthy\n application.\n- Configuration and service failures are observable, attributed, and eligible\n for bounded remediation or rollback.\n- The application continues serving unaffected capabilities when one\n capability is misconfigured or unhealthy.\n- Recovery failure is reported and contained rather than hidden in an\n unbounded retry loop.\n\n## Source changes\n\nAgent-authored source changes occur through the task-execution boundary in an\nisolated workspace. The agent does not directly mutate the running\ninstallation.\n\nPromotion is controlled by deterministic repository-owned policy, including\ntests, generated-reference drift checks, protected paths, change scope, and\nother required gates. The actor proposing a change cannot waive these checks.\n\nRepository and domain separation SHOULD allow a task to load and modify only\nthe governing requirements and code for its declared scope.\n\n## Runtime changes\n\nConfiguration and other runtime changes are proposed as candidates rather than\nmutating the active state in place.\n\nThe affected domain owns:\n\n- the command describing the semantic change;\n- validation of its typed settings;\n- health signals that demonstrate correct operation;\n- domain-specific remediation and rollback consequences.\n\nShared policy supplies consistent evaluation, evidence, and audit mechanics.\nIdentity supplies actor attribution. Infrastructure supplies external\nconfiguration storage, service control, and other operational adapters.\n\n## Runtime change protocol\n\nLast-known-good preservation and candidate promotion are the universal runtime\nchange protocol.\n\n1. An identity proposes an immutable candidate version.\n2. Infrastructure parses it without changing active state.\n3. Each affected domain validates its typed settings.\n4. Shared policy evaluates cross-cutting constraints.\n5. Each affected capability prepares the candidate without destroying its\n active instance.\n6. Capability-specific health checks prove the candidate works.\n7. Runtime supervision atomically promotes the candidate.\n8. A bounded observation period watches for delayed failure.\n9. Failure restores the last-known-good version and records the reason,\n evidence, actor, and involved versions.\n10. Repeated remediation failure stops retrying, isolates the capability, and\n reports the fault.\n\nThe affected domain owns the semantic change command, typed validation, health\nsignals, and remediation consequences. Shared runtime supervision owns the\npromotion transaction. Infrastructure performs storage and service-control\noperations.\n\nThe protocol's exact Go contracts, storage model, atomicity boundary, and\nobservation rules require a focused runtime-supervision design pass.\n\n## Failure isolation\n\nInvalid input MUST leave the active valid configuration unchanged and produce a\ndiagnostic record.\n\nA capability that cannot start or reload SHOULD fail independently where its\nabsence does not violate a declared application invariant. Unaffected\ncapabilities continue operating.\n\nRemediation MUST be bounded, observable, and policy-driven. Recovery MUST NOT\nsilently oscillate between versions or retry forever.\n\n## Database settings: degrade paths and recovery (2026-09)\n\nThe control plane owns runtime-tunable settings and stores them in `archie.db`.\nThis replaced a dashboard overlay store that degraded to file config on any\nfailure and had three documented ways back. The replacement has none, and the\nposture is the opposite: **the daemon fails closed.**\n\n### Degrade paths\n\n| Failure | Boot behaviour | Where the operator sees it |\n| --- | --- | --- |\n| State Store uLine truncated
|
||||
"body": "# Safe Change and Recovery\n\n**Status:** Requirements approved; mechanics are under design. Database-backed settings are implemented through the control plane (2026-09): see the degrade/recovery section below, which records a gap rather than a solution. The full runtime-supervision protocol (bounded observation, versioned audit trail) remains under design.\n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Purpose\n\nArchie is built and improved by agents as well as humans. The architecture MUST\nmake a change easy to locate, understand, test, audit, extend, and reverse\nwithout requiring its author to understand unrelated parts of the application.\n\nSeparation of concerns is therefore an operational safety boundary, not only a\ncode-organisation preference.\n\n## Required outcomes\n\n- An agent can improve a focused feature without receiving authority over\n unrelated domains.\n- Every event, function, action, setting, and policy has an identifiable owner\n and provenance.\n- Source and runtime changes pass deterministic checks outside the actor making\n the change.\n- A rejected or unhealthy candidate cannot replace the last known working\n version.\n- Invalid user configuration does not terminate an otherwise healthy\n application.\n- Configuration and service failures are observable, attributed, and eligible\n for bounded remediation or rollback.\n- The application continues serving unaffected capabilities when one\n capability is misconfigured or unhealthy.\n- Recovery failure is reported and contained rather than hidden in an\n unbounded retry loop.\n\n## Source changes\n\nAgent-authored source changes occur through the task-execution boundary in an\nisolated workspace. The agent does not directly mutate the running\ninstallation.\n\nPromotion is controlled by deterministic repository-owned policy, including\ntests, generated-reference drift checks, protected paths, change scope, and\nother required gates. The actor proposing a change cannot waive these checks.\n\nRepository and domain separation SHOULD allow a task to load and modify only\nthe governing requirements and code for its declared scope.\n\n## Runtime changes\n\nConfiguration and other runtime changes are proposed as candidates rather than\nmutating the active state in place.\n\nThe affected domain owns:\n\n- the command describing the semantic change;\n- validation of its typed settings;\n- health signals that demonstrate correct operation;\n- domain-specific remediation and rollback consequences.\n\nShared policy supplies consistent evaluation, evidence, and audit mechanics.\nIdentity supplies actor attribution. Infrastructure supplies external\nconfiguration storage, service control, and other operational adapters.\n\n## Runtime change protocol\n\nLast-known-good preservation and candidate promotion are the universal runtime\nchange protocol.\n\n1. An identity proposes an immutable candidate version.\n2. Infrastructure parses it without changing active state.\n3. Each affected domain validates its typed settings.\n4. Shared policy evaluates cross-cutting constraints.\n5. Each affected capability prepares the candidate without destroying its\n active instance.\n6. Capability-specific health checks prove the candidate works.\n7. Runtime supervision atomically promotes the candidate.\n8. A bounded observation period watches for delayed failure.\n9. Failure restores the last-known-good version and records the reason,\n evidence, actor, and involved versions.\n10. Repeated remediation failure stops retrying, isolates the capability, and\n reports the fault.\n\nThe affected domain owns the semantic change command, typed validation, health\nsignals, and remediation consequences. Shared runtime supervision owns the\npromotion transaction. Infrastructure performs storage and service-control\noperations.\n\nThe protocol's exact Go contracts, storage model, atomicity boundary, and\nobservation rules require a focused runtime-supervision design pass.\n\n## Failure isolation\n\nInvalid input MUST leave the active valid configuration unchanged and produce a\ndiagnostic record.\n\nA capability that cannot start or reload SHOULD fail independently where its\nabsence does not violate a declared application invariant. Unaffected\ncapabilities continue operating.\n\nRemediation MUST be bounded, observable, and policy-driven. Recovery MUST NOT\nsilently oscillate between versions or retry forever.\n\n## Database settings: degrade paths and recovery (2026-09)\n\nThe control plane owns runtime-tunable settings and stores them in `archie.db`.\nThis replaced a dashboard overlay store that degraded to file config on any\nfailure and had three documented ways back. The replacement has none, and the\nposture is the opposite: **the daemon fails closed.**\n\n### Degrade paths\n\n| Failure | Boot behaviour | Where the operator sees it |\n| --- | --- | --- |\n| State Store uLine truncated
|
||||
},
|
||||
{
|
||||
"url": "/dev/prds/01-project-management/",
|
||||
|
||||
@@ -5,12 +5,12 @@
|
||||
// State Store is up -- is unavailable in exactly the case that needs it
|
||||
// (docs/architecture/safe-change-and-recovery.md).
|
||||
//
|
||||
// Every operation works on the task database file directly, with the State
|
||||
// Store stopped, and the ones that write refuse when another process owns the
|
||||
// file. Backup is the exception and deliberately so: the update installer
|
||||
// cannot stop the process that runs it, so the snapshot that makes a failed
|
||||
// update reversible is taken against a serving store through SQLite's own
|
||||
// consistent copy.
|
||||
// With -db an operation works on the legacy SQLite task file; without it, on
|
||||
// the PostgreSQL database database_url names (state_store_recovery_postgres.go).
|
||||
// The ones that write refuse while a serving process owns the store. Backup is
|
||||
// the exception and deliberately so: the update installer cannot stop the
|
||||
// process that runs it, so the snapshot that makes a failed update recoverable
|
||||
// is taken against a serving store through a consistent copy.
|
||||
//
|
||||
// The control-plane server both validate and rollback read through is built by
|
||||
// openStateStoreControlPlane -- the same root helper the served path uses -- so
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
@@ -76,31 +77,33 @@ type StateStoreRecoveryOptions struct {
|
||||
// RunStateStoreRecovery performs one offline operation on the task database and
|
||||
// returns the line the command reports on stdout.
|
||||
func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
|
||||
// -db names the legacy SQLite task file; without it the command works on
|
||||
// the PostgreSQL database the configuration names.
|
||||
if options.DB == "" {
|
||||
return runPostgresRecovery(ctx, options)
|
||||
}
|
||||
switch options.Operation {
|
||||
case RecoveryBackup:
|
||||
if options.DB == "" || options.Out == "" {
|
||||
return "", errors.New("backup requires -db and -out")
|
||||
if options.Out == "" {
|
||||
return "", errors.New("backup requires -out")
|
||||
}
|
||||
if err := store.Backup(ctx, options.DB, options.Out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("backed up %s to %s", options.DB, options.Out), nil
|
||||
case RecoveryRestore:
|
||||
if options.DB == "" || options.From == "" {
|
||||
return "", errors.New("restore requires -db and -from")
|
||||
if options.From == "" {
|
||||
return "", errors.New("restore requires -from")
|
||||
}
|
||||
if err := store.Restore(ctx, options.DB, options.From); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("restored %s from %s; start the State Store again", options.DB, options.From), nil
|
||||
case RecoveryValidate:
|
||||
if options.DB == "" {
|
||||
return "", errors.New("validate requires -db")
|
||||
}
|
||||
return validateStore(ctx, options)
|
||||
case RecoveryRollback:
|
||||
if options.DB == "" || options.Kind == "" {
|
||||
return "", errors.New("rollback requires -db and -kind")
|
||||
if options.Kind == "" {
|
||||
return "", errors.New("rollback requires -kind")
|
||||
}
|
||||
return rollbackResource(ctx, options)
|
||||
default:
|
||||
@@ -146,34 +149,45 @@ func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (stri
|
||||
// with the writer's validator, and boot's gate is the config check alone.
|
||||
// That is the same stance the layering takes for a kind the table is missing:
|
||||
// what the daemon reads for a kind nothing was ever stored for is the seed.
|
||||
checked := 0
|
||||
stored, err := st.StoresResources(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if stored {
|
||||
if checked, err = server.ValidateStored(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
base, err := bootConfig(ctx, options)
|
||||
checked, err := checkBootGate(ctx, server, stored, options)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", options.DB, version, checked), nil
|
||||
}
|
||||
|
||||
// checkBootGate runs the writer's per-resource validation over what is stored,
|
||||
// then boot's gate over the file config with the stored resources layered on,
|
||||
// and returns how many stored resources validated.
|
||||
func checkBootGate(ctx context.Context, server *controlplane.Server, stored bool, options StateStoreRecoveryOptions) (int, error) {
|
||||
checked := 0
|
||||
if stored {
|
||||
var err error
|
||||
if checked, err = server.ValidateStored(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
base, err := bootConfig(ctx, options)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
document := base
|
||||
if stored {
|
||||
document, _, err = server.StoredRuntimeConfig(ctx, base)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := configuration.Validate(&document); err != nil {
|
||||
// The daemon's own wording, so an operator can match this failure to the
|
||||
// one that stopped archied starting.
|
||||
return "", fmt.Errorf("validate database settings: %w", err)
|
||||
return 0, fmt.Errorf("validate database settings: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", options.DB, version, checked), nil
|
||||
return checked, nil
|
||||
}
|
||||
|
||||
// bootConfig resolves the configuration the daemon would boot with. validate is
|
||||
@@ -217,22 +231,27 @@ func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (s
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = st.Close() }()
|
||||
return replayRevision(ctx, st, options)
|
||||
}
|
||||
|
||||
server, err := openStateStoreControlPlane(st)
|
||||
// replayRevision puts back the revision options names through the control
|
||||
// plane's ordinary replace. The caller holds the store's ownership claim.
|
||||
func replayRevision(ctx context.Context, resources controlplane.ResourceStore, options StateStoreRecoveryOptions) (string, error) {
|
||||
server, err := openStateStoreControlPlane(resources)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !server.Owns(options.Kind) {
|
||||
return "", fmt.Errorf("unknown resource kind %q", options.Kind)
|
||||
}
|
||||
current, err := st.Resource(ctx, options.Kind)
|
||||
current, err := resources.Resource(ctx, options.Kind)
|
||||
if errors.Is(err, store.ErrResourceNotFound) {
|
||||
return "", fmt.Errorf("%s has no stored value to roll back", options.Kind)
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s: %w", options.Kind, err)
|
||||
}
|
||||
revision, err := revisionToReplay(ctx, st, options.Kind, current.Version, options.Revision)
|
||||
revision, err := revisionToReplay(ctx, resources, options.Kind, current.Version, options.Revision)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -254,7 +273,7 @@ func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (s
|
||||
|
||||
// revisionToReplay picks the value to put back: the revision the operator
|
||||
// named, or the newest one older than the value the resource carries now.
|
||||
func revisionToReplay(ctx context.Context, st *store.Store, kind string, current, requested int64) (store.Resource, error) {
|
||||
func revisionToReplay(ctx context.Context, st controlplane.ResourceStore, kind string, current, requested int64) (store.Resource, error) {
|
||||
history, err := st.ResourceHistory(ctx, kind, 0)
|
||||
if err != nil {
|
||||
return store.Resource{}, fmt.Errorf("read %s history: %w", kind, err)
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
|
||||
)
|
||||
|
||||
// runPostgresRecovery performs one recovery operation on the PostgreSQL
|
||||
// database the configuration names. There is no automatic schema rollback:
|
||||
// goose Down drops tables, so the way back from a migration is restoring the
|
||||
// snapshot taken before it, which loses every write made after that snapshot.
|
||||
func runPostgresRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
|
||||
cfg, err := bootConfig(ctx, options)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
url := cfg.DatabaseURL
|
||||
if url == "" {
|
||||
return "", errors.New("database_url is required: the configuration names no PostgreSQL database (pass -db for a legacy SQLite task file)")
|
||||
}
|
||||
switch options.Operation {
|
||||
case RecoveryBackup:
|
||||
if options.Out == "" {
|
||||
return "", errors.New("backup requires -out")
|
||||
}
|
||||
if err := postgres.Backup(ctx, url, options.Out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "backed up the whole database to " + options.Out, nil
|
||||
case RecoveryRestore:
|
||||
if options.From == "" {
|
||||
return "", errors.New("restore requires -from")
|
||||
}
|
||||
if err := postgres.Restore(ctx, url, options.From); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("restored the whole database from %s; every write made after the snapshot is gone; start the archie services again", options.From), nil
|
||||
case RecoveryValidate:
|
||||
return validatePostgres(ctx, url, options)
|
||||
case RecoveryRollback:
|
||||
if options.Kind == "" {
|
||||
return "", errors.New("rollback requires -kind")
|
||||
}
|
||||
return rollbackPostgres(ctx, url, options)
|
||||
default:
|
||||
return "", fmt.Errorf("unknown recovery command %q", options.Operation)
|
||||
}
|
||||
}
|
||||
|
||||
// openRecoveryPool opens the database without migrating it: recovery inspects
|
||||
// or repairs the schema it finds, and refuses one newer than this binary.
|
||||
func openRecoveryPool(ctx context.Context, url string) (*pgxpool.Pool, int64, error) {
|
||||
pool, err := postgres.Open(ctx, url)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
version, err := postgres.SchemaVersion(ctx, pool)
|
||||
if err == nil {
|
||||
err = postgres.CheckSchemaVersion(version)
|
||||
}
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
return nil, 0, err
|
||||
}
|
||||
return pool, version, nil
|
||||
}
|
||||
|
||||
// validatePostgres is validateStore for the PostgreSQL database. An
|
||||
// unmigrated database holds nothing to check; the State Store migrates and
|
||||
// seeds it on its next start, so only boot's config gate applies.
|
||||
func validatePostgres(ctx context.Context, url string, options StateStoreRecoveryOptions) (string, error) {
|
||||
pool, version, err := openRecoveryPool(ctx, url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer pool.Close()
|
||||
server, err := openStateStoreControlPlane(postgres.NewResources(pool))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
checked, err := checkBootGate(ctx, server, version > 0, options)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("the database is valid at schema version %d; %d stored resources validate", version, checked), nil
|
||||
}
|
||||
|
||||
// rollbackPostgres replays an earlier revision holding the State Store's
|
||||
// ownership claim, so it refuses while a State Store serves and none can
|
||||
// start until it finishes.
|
||||
func rollbackPostgres(ctx context.Context, url string, options StateStoreRecoveryOptions) (string, error) {
|
||||
pool, _, err := openRecoveryPool(ctx, url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer pool.Close()
|
||||
claim, err := postgres.AcquireOwnership(ctx, pool, postgres.OwnerStateStore)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("rollback refused, stop the State Store first: %w", err)
|
||||
}
|
||||
defer func() { _ = claim.Release(context.WithoutCancel(ctx)) }()
|
||||
return replayRevision(ctx, postgres.NewResources(pool), options)
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
|
||||
)
|
||||
|
||||
// pgRecoveryFixture is a migrated database and the config file naming it,
|
||||
// which is all the Postgres recovery path is given: it resolves database_url
|
||||
// from the configuration exactly as the State Store does.
|
||||
func pgRecoveryFixture(t *testing.T) (configPath string, pool *pgxpool.Pool, url string) {
|
||||
t.Helper()
|
||||
url = pgtest.URL(t)
|
||||
dir := t.TempDir()
|
||||
configPath = filepath.Join(dir, "config.toml")
|
||||
body := fmt.Sprintf("bot_user = 'archie'\ndb_path = %q\ndatabase_url = %q\n[forge]\ntype = 'github'\nhost = 'https://github.example.com'\n",
|
||||
filepath.Join(dir, "archie.db"), url)
|
||||
if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pool, err := postgres.Open(t.Context(), url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
if err := postgres.Migrate(t.Context(), pool, postgres.Migrations()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return configPath, pool, url
|
||||
}
|
||||
|
||||
func requirePGTools(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, tool := range []string{"pg_dump", "pg_restore"} {
|
||||
if _, err := exec.LookPath(tool); err != nil {
|
||||
t.Skipf("%s is not on PATH; this test needs the PostgreSQL 18 client tools", tool)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func putResource(t *testing.T, pool *pgxpool.Pool, kind, value, label string) {
|
||||
t.Helper()
|
||||
resources := postgres.NewResources(pool)
|
||||
expected := int64(0)
|
||||
if current, err := resources.Resource(t.Context(), kind); err == nil {
|
||||
expected = current.Version
|
||||
}
|
||||
if _, err := resources.PutResource(t.Context(), storecontract.ResourceWrite{
|
||||
Kind: kind, Value: []byte(value), Actor: label, Source: "test",
|
||||
RequestID: fmt.Sprintf("%s-%d", label, expected+1), ExpectedVersion: expected,
|
||||
}); err != nil {
|
||||
t.Fatalf("put %s: %v", kind, err)
|
||||
}
|
||||
}
|
||||
|
||||
func holdRole(t *testing.T, url, role string) {
|
||||
t.Helper()
|
||||
pool, err := postgres.Open(t.Context(), url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
claim, err := postgres.AcquireOwnership(t.Context(), pool, role)
|
||||
if err != nil {
|
||||
t.Fatalf("hold %s: %v", role, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = claim.Release(t.Context()) })
|
||||
}
|
||||
|
||||
func TestPostgresRecoveryRequiresADatabaseURL(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath := filepath.Join(dir, "config.toml")
|
||||
if err := os.WriteFile(configPath, []byte("bot_user = 'archie'\n[forge]\ntype = 'github'\nhost = 'https://github.example.com'\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, op := range []string{RecoveryBackup, RecoveryRestore, RecoveryValidate, RecoveryRollback} {
|
||||
t.Run(op, func(t *testing.T) {
|
||||
_, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{
|
||||
Operation: op, Config: configPath, Out: filepath.Join(dir, "o"), From: filepath.Join(dir, "f"), Kind: "k",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "database_url") {
|
||||
t.Fatalf("%s without database_url = %v, want a refusal naming database_url", op, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresRecoveryBackupAndRestore(t *testing.T) {
|
||||
requirePGTools(t)
|
||||
configPath, pool, url := pgRecoveryFixture(t)
|
||||
putResource(t, pool, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "snapshot")
|
||||
snapshot := filepath.Join(t.TempDir(), "snapshot.dump")
|
||||
|
||||
if _, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryBackup, Config: configPath, Out: snapshot}); err != nil {
|
||||
t.Fatalf("backup: %v", err)
|
||||
}
|
||||
putResource(t, pool, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "later")
|
||||
|
||||
for _, role := range postgres.ServiceOwners {
|
||||
t.Run("refused while "+role+" serves", func(t *testing.T) {
|
||||
holdRole(t, url, role)
|
||||
_, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryRestore, Config: configPath, From: snapshot})
|
||||
if !errors.Is(err, postgres.ErrOwned) {
|
||||
t.Fatalf("restore with %s serving = %v, want ErrOwned", role, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
summary, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryRestore, Config: configPath, From: snapshot})
|
||||
if err != nil {
|
||||
t.Fatalf("restore: %v", err)
|
||||
}
|
||||
if !strings.Contains(summary, "after the snapshot") {
|
||||
t.Errorf("restore must say writes after the snapshot are lost; summary = %q", summary)
|
||||
}
|
||||
current, err := postgres.NewResources(pool).Resource(t.Context(), controlplane.ModelRoleAssignmentsKind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if current.Version != 1 {
|
||||
t.Errorf("restored version = %d, want the snapshot's 1", current.Version)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresRecoveryValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(t *testing.T, pool *pgxpool.Pool)
|
||||
wantErr string
|
||||
}{
|
||||
{name: "healthy store", setup: func(*testing.T, *pgxpool.Pool) {}},
|
||||
{name: "schema newer than this binary", wantErr: "newer than this binary", setup: func(t *testing.T, pool *pgxpool.Pool) {
|
||||
if _, err := pool.Exec(t.Context(), "INSERT INTO goose_db_version (version_id, is_applied) VALUES ($1, true)", postgres.SupportedSchemaVersion()+1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "stored value boot refuses", wantErr: "dispatch.trigger", setup: func(t *testing.T, pool *pgxpool.Pool) {
|
||||
putResource(t, pool, "scheduling-policy", `{"poll_interval":"1m","max_retries":3,"dispatch":{"trigger":"bogus"}}`, "bad")
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
configPath, pool, _ := pgRecoveryFixture(t)
|
||||
tt.setup(t, pool)
|
||||
summary, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryValidate, Config: configPath})
|
||||
if tt.wantErr == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("validate: %v", err)
|
||||
}
|
||||
if !strings.Contains(summary, fmt.Sprintf("schema version %d", postgres.SupportedSchemaVersion())) {
|
||||
t.Errorf("summary = %q, want the schema version", summary)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("validate = %v, want refusal containing %q", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresRecoveryRollback(t *testing.T) {
|
||||
configPath, pool, url := pgRecoveryFixture(t)
|
||||
kind := controlplane.ModelRoleAssignmentsKind
|
||||
putResource(t, pool, kind, `{"implement":"anthropic/claude"}`, "first")
|
||||
putResource(t, pool, kind, `{"implement":"openai/gpt"}`, "second")
|
||||
options := StateStoreRecoveryOptions{Operation: RecoveryRollback, Config: configPath, Kind: kind}
|
||||
|
||||
t.Run("refused while the State Store serves", func(t *testing.T) {
|
||||
holdRole(t, url, postgres.OwnerStateStore)
|
||||
if _, err := RunStateStoreRecovery(t.Context(), options); !errors.Is(err, postgres.ErrOwned) {
|
||||
t.Fatalf("rollback with the State Store serving = %v, want ErrOwned", err)
|
||||
}
|
||||
})
|
||||
|
||||
if _, err := RunStateStoreRecovery(t.Context(), options); err != nil {
|
||||
t.Fatalf("rollback: %v", err)
|
||||
}
|
||||
current, err := postgres.NewResources(pool).Resource(t.Context(), kind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(current.Value) != `{"implement": "anthropic/claude"}` && string(current.Value) != `{"implement":"anthropic/claude"}` {
|
||||
t.Errorf("value = %s, want the first revision's", current.Value)
|
||||
}
|
||||
if current.Version != 3 {
|
||||
t.Errorf("version = %d, want the rollback recorded as revision 3", current.Version)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const snapshotConfig = "database_url = 'postgres://archie:hunter2@db.example/archie'\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n"
|
||||
|
||||
// The new State Store migrates on start and nothing undoes a migration
|
||||
// automatically, so a daemon update snapshots the whole database first,
|
||||
// through the release's own recovery subcommand, and hands the snapshot's
|
||||
// path to the watchdog. The connection string stays inside the config.
|
||||
func TestUpdateInstallSnapshotsTheDatabaseBeforeMigrating(t *testing.T) {
|
||||
snapshot := filepath.Join(t.TempDir(), "pre-update.dump")
|
||||
_, calls, _ := runUpdateInstallWithConfig(t, map[string]string{
|
||||
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
||||
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
||||
"ARCHIE_DB_SNAPSHOT": snapshot,
|
||||
}, snapshotConfig, false)
|
||||
|
||||
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-config ", "-out "+snapshot)
|
||||
assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_DB_SNAPSHOT="+snapshot)
|
||||
assertCallAbsent(t, calls, "hunter2")
|
||||
}
|
||||
|
||||
func TestUpdateInstallSnapshotScope(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
wantSnapshot bool
|
||||
wantRefusal bool
|
||||
}{
|
||||
{name: "agent-only update does not touch the database", env: map[string]string{
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", "ARCHIE_UPDATE_AGENT_VERSION": "1.22.0",
|
||||
}},
|
||||
{name: "daemon update refuses when the snapshot fails", wantSnapshot: true, wantRefusal: true, env: map[string]string{
|
||||
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0", "ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", "ARCHIE_TEST_GO_FAILS_SNAPSHOT": "1",
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, calls, output := runUpdateInstallWithConfig(t, tt.env, snapshotConfig, tt.wantRefusal)
|
||||
snapshotted := false
|
||||
for _, call := range calls {
|
||||
if strings.Contains(call, "archie-state-store backup") && strings.Contains(call, "-config ") {
|
||||
snapshotted = true
|
||||
}
|
||||
}
|
||||
if snapshotted != tt.wantSnapshot {
|
||||
t.Fatalf("snapshot attempted = %v, want %v; calls = %q", snapshotted, tt.wantSnapshot, calls)
|
||||
}
|
||||
if !tt.wantRefusal {
|
||||
return
|
||||
}
|
||||
if !strings.Contains(output, "could not snapshot the database") {
|
||||
t.Errorf("refusal must say the snapshot failed; output = %q", output)
|
||||
}
|
||||
for _, call := range calls {
|
||||
if strings.Contains(call, "install -m755") && !strings.Contains(call, "/scripts/") {
|
||||
t.Errorf("refused update still installed %q", call)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A failed daemon update rolls the binaries back but never the database, and
|
||||
// says so, naming the explicit offline restore rather than running one.
|
||||
func TestUpdateWatchdogNeverRestoresTheDatabaseItself(t *testing.T) {
|
||||
root, err := filepath.Abs(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
work := t.TempDir()
|
||||
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
|
||||
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, content := range map[string]string{"archied": "new", "archied.prev": "old"} {
|
||||
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
callsPath := filepath.Join(work, "calls")
|
||||
writeFakeCommand(t, fakeDir, "systemctl", `exit 0`)
|
||||
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
|
||||
for _, tool := range []string{"pg_restore", "psql", "go", "archie-state-store"} {
|
||||
writeFakeCommand(t, fakeDir, tool, `printf '%s\n' "`+tool+` $*" >> "$ARCHIE_TEST_CALLS"`)
|
||||
}
|
||||
snapshot := filepath.Join(work, "pre-update.dump")
|
||||
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
|
||||
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
|
||||
"ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_REPORT_PATH="+filepath.Join(work, "report.json"),
|
||||
"ARCHIE_UPDATE_HEALTH_TIMEOUT=0", "ARCHIE_UPDATE_COMPONENTS=daemon",
|
||||
"ARCHIE_DB_SNAPSHOT="+snapshot, "ARCHIE_CONFIG_PATH=/etc/archie/config.toml")
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
||||
}
|
||||
if calls, _ := os.ReadFile(callsPath); len(calls) != 0 {
|
||||
t.Errorf("watchdog ran database tooling itself: %q", calls)
|
||||
}
|
||||
for _, want := range []string{"NOT rolled back", "archie-state-store restore -config /etc/archie/config.toml -from " + snapshot, "lost"} {
|
||||
if !strings.Contains(string(output), want) {
|
||||
t.Errorf("output lacks %q:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -411,6 +411,8 @@ printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
|
||||
# must refuse the update.
|
||||
if [ "$1" = "run" ]; then
|
||||
[ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1
|
||||
# The PostgreSQL snapshot is the backup run that names no -db file.
|
||||
[ "${ARCHIE_TEST_GO_FAILS_SNAPSHOT:-}" = "1" ] && [[ " $* " != *" -db "* ]] && exit 1
|
||||
exit 0
|
||||
fi
|
||||
out=""
|
||||
|
||||
@@ -80,6 +80,7 @@ AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
|
||||
AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}"
|
||||
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}"
|
||||
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
|
||||
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
|
||||
# The address the daemon dials for its state, so the watchdog can wait for it to
|
||||
# accept connections before it starts archied. Read with the same narrow awk as
|
||||
# the task database path and for the same reason: the daemon stays the authority
|
||||
@@ -284,6 +285,22 @@ if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_
|
||||
fi
|
||||
fi
|
||||
|
||||
# The pre-migration snapshot of the PostgreSQL database: the whole database,
|
||||
# since every service shares it. The new State Store migrates on start and a
|
||||
# migration is never undone automatically (goose Down drops tables), so this
|
||||
# snapshot is the only way back, restored by an operator with every service
|
||||
# stopped. The recovery subcommand resolves database_url from the same config
|
||||
# the services boot with, so the connection string never passes through this
|
||||
# script. No snapshot, no update.
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
echo "==> snapshotting the database before the update migrates it"
|
||||
mkdir -p "$(dirname "$DB_SNAPSHOT")"
|
||||
if ! go run ./cmd/archie-state-store backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT"; then
|
||||
echo "refusing to install $GW: could not snapshot the database named by $CONFIG_PATH, so a failed migration could not be recovered" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Refresh adapters from the approved release source. The running shell
|
||||
# continues with this copy; subsequent updates use the current contract.
|
||||
install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install"
|
||||
@@ -377,6 +394,8 @@ systemd-run --user --unit=archied-update-restart --collect --quiet \
|
||||
--setenv=ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS="${AGENT_VERSION_PREVIOUS_EXISTS:-0}" \
|
||||
--setenv=ARCHIE_TASK_DB_PATH="$TASK_DB_PATH" \
|
||||
--setenv=ARCHIE_TASK_DB_BACKUP="$TASK_DB_BACKUP" \
|
||||
--setenv=ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT" \
|
||||
--setenv=ARCHIE_CONFIG_PATH="$CONFIG_PATH" \
|
||||
--setenv=ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET" \
|
||||
"$BIN_DIR/archie-update-watchdog"
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@ AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.versi
|
||||
AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}"
|
||||
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}"
|
||||
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
|
||||
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
|
||||
CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}"
|
||||
# The daemon exports its own health address (releaseupdate.CommandInstaller);
|
||||
# this default only covers a hand-driven run, and must stay the address archied
|
||||
# itself serves /healthz on -- [health] listen, defaulted in
|
||||
@@ -218,6 +220,16 @@ rollback_task_db() {
|
||||
rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm"
|
||||
}
|
||||
|
||||
# explain_database_recovery states what a daemon rollback does not do. The
|
||||
# PostgreSQL database is never restored automatically: goose Down drops tables,
|
||||
# and a restore is destructive, loses every write after the snapshot and needs
|
||||
# every service stopped, so it is an operator's decision, not this script's.
|
||||
explain_database_recovery() {
|
||||
echo "==> the PostgreSQL database was NOT rolled back; schema migrations are never undone automatically" >&2
|
||||
echo "==> if the previous release refuses the migrated schema, stop every archie service and restore the pre-update snapshot (every write made after it is lost):" >&2
|
||||
echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2
|
||||
}
|
||||
|
||||
start_units
|
||||
|
||||
if wait_healthy; then
|
||||
@@ -273,5 +285,8 @@ fi
|
||||
if [ "$rolled_back" = true ]; then
|
||||
start_units
|
||||
fi
|
||||
if [[ ",$COMPONENTS," == *,daemon,* ]]; then
|
||||
explain_database_recovery
|
||||
fi
|
||||
|
||||
write_report "failed" "$rolled_back"
|
||||
Reference in new issue
Block a user