From bf7115fcb5efefa8221cdd9b4eb3fb87622c0f0d Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 3 Sep 2026 14:53:03 +1000 Subject: [PATCH] fix(update): make daemon rollback schema-safe (#704) --- internal/app/archied/bootstrap.go | 23 +++-- internal/app/archied/bootstrap_nats_test.go | 37 ++++++++ internal/releaseupdate/install_script_test.go | 68 +++++++++++++++ scripts/archie-update-install | 23 +++++ scripts/archie-update-watchdog | 84 ++++++++++++------- 5 files changed, 198 insertions(+), 37 deletions(-) diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 7b9b58c27..069bf35b9 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -370,7 +370,6 @@ func (b *boot) setupObservability() { // doesn't expose it. See docs/prds/payload-field-mapping.md. if ms, ok := b.st.(store.MappingStore); ok { b.web.Mappings = ms - b.d.Mappings = ms } else { log.Warn("mapping storage unavailable: task store does not implement MappingStore") } @@ -383,21 +382,14 @@ func (b *boot) setupObservability() { // See docs/prds/webhook-intake-security.md. if bs, ok := b.st.(store.BindingStore); ok { b.web.Bindings = bs - b.d.Bindings = bs } else { log.Warn("binding storage unavailable: task store does not implement BindingStore") } if bd, ok := b.st.(store.BindingDispatcher); ok { b.web.BindingDispatcher = bd - b.d.BindingDispatcher = bd } else { log.Warn("binding dispatcher unavailable: task store does not implement BindingDispatcher") } - if btc, ok := b.st.(store.BindingTaskCreator); ok { - b.d.BindingTaskCreator = btc - } else { - log.Warn("binding task creator unavailable: task store does not implement BindingTaskCreator") - } sink := bus.Subscribe(256) go persistAndBroadcastEvents(sink, b.st, b.web, log) } @@ -411,6 +403,9 @@ func (b *boot) connectNATS(ctx context.Context) error { url := cfg.NATS.URL var natsToken string if cfg.NATS.Mode == config.NATSModeExternal { + if url == "" { + return fmt.Errorf("nats.url is required when nats.mode is external") + } var err error natsToken, err = configuredNATSToken(cfg.NATS, os.Getenv) if err != nil { @@ -1242,6 +1237,18 @@ func (b *boot) buildDaemon() { // this point /api/config and the daemon can never disagree about the // published config. b.web.Cfg = b.d.Cfg + if ms, ok := b.st.(store.MappingStore); ok { + b.d.Mappings = ms + } + if bs, ok := b.st.(store.BindingStore); ok { + b.d.Bindings = bs + } + if bd, ok := b.st.(store.BindingDispatcher); ok { + b.d.BindingDispatcher = bd + } + if btc, ok := b.st.(store.BindingTaskCreator); ok { + b.d.BindingTaskCreator = btc + } b.setupForgeWebhook() } diff --git a/internal/app/archied/bootstrap_nats_test.go b/internal/app/archied/bootstrap_nats_test.go index 1ee690b0d..7e8704e66 100644 --- a/internal/app/archied/bootstrap_nats_test.go +++ b/internal/app/archied/bootstrap_nats_test.go @@ -150,3 +150,40 @@ func writeDockerResponse(t *testing.T, w http.ResponseWriter, value any) { t.Errorf("encode Docker response: %v", err) } } + +func TestConnectExternalNATSEmptyURLReturnsConfigurationError(t *testing.T) { + b := newBootstrap() + b.cfg.NATS = config.NATSConfig{Mode: config.NATSModeExternal} + + err := b.connectNATS(t.Context()) + if err == nil || !strings.Contains(err.Error(), "nats.url is required") { + t.Fatalf("connectNATS() error = %v, want useful missing-url error", err) + } +} + +func TestSetupObservabilityDoesNotTouchDaemonBeforeBuild(t *testing.T) { + fileset := token.NewFileSet() + file, err := parser.ParseFile(fileset, "bootstrap.go", nil, 0) + if err != nil { + t.Fatal(err) + } + setup := methodBody(t, file, "setupObservability") + var daemonAssignments int + ast.Inspect(setup, func(node ast.Node) bool { + assign, ok := node.(*ast.AssignStmt) + if !ok { + return true + } + for _, lhs := range assign.Lhs { + if selector, ok := lhs.(*ast.SelectorExpr); ok { + if receiver, ok := selector.X.(*ast.SelectorExpr); ok && receiver.Sel.Name == "d" { + daemonAssignments++ + } + } + } + return true + }) + if daemonAssignments != 0 { + t.Fatalf("setupObservability assigns daemon fields %d times; daemon is built later", daemonAssignments) + } +} diff --git a/internal/releaseupdate/install_script_test.go b/internal/releaseupdate/install_script_test.go index 02955fe04..1b7b2fae1 100644 --- a/internal/releaseupdate/install_script_test.go +++ b/internal/releaseupdate/install_script_test.go @@ -3,6 +3,7 @@ package releaseupdate import ( "bufio" "encoding/json" + "fmt" "os" "os/exec" "path/filepath" @@ -121,6 +122,73 @@ func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponent } } +func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + work := t.TempDir() + binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin") + reportPath := filepath.Join(work, "report.json") + dbPath, backupPath := filepath.Join(work, "tasks.sqlite"), filepath.Join(work, "tasks.prev.sqlite") + if err := os.MkdirAll(binDir, 0o755); err != nil { + t.Fatal(err) + } + for name, content := range map[string]string{"archied": "new", "archied.prev": "old"} { + if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil { + t.Fatal(err) + } + } + for path, version := range map[string]int{dbPath: 2, backupPath: 1} { + if output, err := exec.Command("sqlite3", path, fmt.Sprintf("PRAGMA user_version=%d; CREATE TABLE tasks (id INTEGER);", version)).CombinedOutput(); err != nil { + t.Fatalf("create database %s: %v: %s", path, err, output) + } + } + callsPath := filepath.Join(work, "calls") + writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`) + writeFakeCommand(t, fakeDir, "cp", `printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"; /usr/bin/cp "$@"`) + writeFakeCommand(t, fakeDir, "curl", `exit 1`) + cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog")) + cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir, + "ARCHIE_TEST_CALLS="+callsPath, + "ARCHIE_UPDATE_REPORT_PATH="+reportPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", "ARCHIE_UPDATE_COMPONENTS=daemon", + "ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.19.10", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.20.0", + "ARCHIE_TASK_DB_PATH="+dbPath, "ARCHIE_TASK_DB_BACKUP="+backupPath) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("watchdog failed: %v\n%s", err, output) + } + output, err := exec.Command("sqlite3", dbPath, "PRAGMA user_version;").Output() + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(string(output)) != "1" { + t.Fatalf("restored schema version = %q, want 1", output) + } + calls, err := os.ReadFile(callsPath) + if err != nil { + t.Fatal(err) + } + callLines := strings.Split(strings.TrimSpace(string(calls)), "\n") + stopIndex, copyIndex, startIndex := -1, -1, -1 + for i, line := range callLines { + switch line { + case "systemctl --user stop archied.service": + stopIndex = i + case "systemctl --user restart archied.service": + startIndex = i + } + if strings.HasPrefix(line, "cp ") && strings.Contains(line, "tasks.prev.sqlite") { + copyIndex = i + } + } + if stopIndex < 0 || copyIndex < 0 || stopIndex > copyIndex { + t.Fatalf("rollback must stop service before restoring database; calls = %q", calls) + } + if startIndex < 0 || startIndex <= copyIndex { + t.Fatalf("rollback must restart service after restoring database; calls = %q", calls) + } +} + func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) { t.Helper() ctx := t.Context() diff --git a/scripts/archie-update-install b/scripts/archie-update-install index 03e9efddc..9722d950b 100755 --- a/scripts/archie-update-install +++ b/scripts/archie-update-install @@ -43,9 +43,24 @@ configured_agent_image() { printf '%s' "$image" } +configured_task_db_path() { + awk ' + /^[[:space:]]*db_path[[:space:]]*=/ { + value = $0 + sub(/^[[:space:]]*db_path[[:space:]]*=[[:space:]]*/, "", value) + sub(/[[:space:]]*(#.*)?$/, "", value) + if (value ~ /^".*"$/) { sub(/^"/, "", value); sub(/"$/, "", value); print value } + else if (length(value) >= 2 && substr(value, 1, 1) == sprintf("%c", 39) && substr(value, length(value), 1) == sprintf("%c", 39)) print substr(value, 2, length(value) - 2) + exit + } + ' "$CONFIG_PATH" 2>/dev/null || true +} + AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-$(configured_agent_image)}" AGENT_IMAGE="${AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}" AGENT_IMAGE_BACKUP="$BIN_DIR/archie-agent-image.prev.tar" +TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}" +TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}" PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}" PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}" @@ -124,6 +139,12 @@ install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-updat install -m755 "$source_dir/scripts/archie-update-watchdog" "$BIN_DIR/archie-update-watchdog" install -m755 "$source_dir/scripts/archie-update-check" "$BIN_DIR/archie-update-check" +if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_DB_PATH" ]; then + echo "==> backing up task database for schema-safe rollback" + mkdir -p "$(dirname "$TASK_DB_BACKUP")" + sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'" +fi + if [[ ",$components," == *,daemon,* ]]; then echo "==> building archied $GW" go build -ldflags "-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=$GW -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=$RT -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" \ @@ -157,6 +178,8 @@ systemd-run --user --unit=archied-update-restart --collect --quiet \ --setenv=ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT" \ --setenv=ARCHIE_AGENT_IMAGE="$AGENT_IMAGE" \ --setenv=ARCHIE_AGENT_IMAGE_BACKUP="$AGENT_IMAGE_BACKUP" \ + --setenv=ARCHIE_TASK_DB_PATH="$TASK_DB_PATH" \ + --setenv=ARCHIE_TASK_DB_BACKUP="$TASK_DB_BACKUP" \ "$BIN_DIR/archie-update-watchdog" case "$components" in diff --git a/scripts/archie-update-watchdog b/scripts/archie-update-watchdog index da61ec855..1a168b0f7 100755 --- a/scripts/archie-update-watchdog +++ b/scripts/archie-update-watchdog @@ -10,6 +10,8 @@ set -uo pipefail BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}" AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}" AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}" +TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}" +TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}" HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}" REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}" @@ -82,6 +84,20 @@ rollback_agent_image() { docker image load --input "$AGENT_IMAGE_BACKUP" } +rollback_task_db() { + if [ -z "$TASK_DB_PATH" ]; then + echo "==> no task database path; cannot restore schema-compatible state" >&2 + return 1 + fi + if [ ! -f "$TASK_DB_BACKUP" ]; then + echo "==> no task database backup; cannot restore schema-compatible state" >&2 + return 1 + fi + sqlite3 "$TASK_DB_BACKUP" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true + cp -f "$TASK_DB_BACKUP" "$TASK_DB_PATH" + rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm" +} + echo "==> restarting archied.service" systemctl --user restart archied.service @@ -93,36 +109,46 @@ fi echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components" rolled_back=false -case "$COMPONENTS" in - daemon) - if [ -f "$BIN_DIR/archied.prev" ]; then - rollback_component archied && rolled_back=true - else - echo "==> no backup for archied; cannot roll it back automatically" >&2 - fi - ;; - agent) - if [ -f "$AGENT_IMAGE_BACKUP" ]; then - rollback_agent_image && rolled_back=true - else - echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2 - fi - ;; - daemon,agent) - # Preserve the old all-or-nothing rollback guarantee: never restore one - # component and leave the other on the failed release. - if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ]; then - if rollback_agent_image && rollback_component archied; then - rolled_back=true +# Stop before touching either the executable or its database. The service +# uses Restart=on-failure in the supported unit, so copying rollback state +# while a crash-loop child is alive can race with SQLite and execute the new +# binary after its files have been replaced. +if systemctl --user stop archied.service; then + case "$COMPONENTS" in + daemon) + if [ -f "$BIN_DIR/archied.prev" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then + if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then + rolled_back=true + fi + else + echo "==> no backup for archied; cannot roll it back automatically" >&2 fi - else - echo "==> not all changed-component backups exist; cannot roll back automatically" >&2 - fi - ;; - *) - echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2 - ;; -esac + ;; + agent) + if [ -f "$AGENT_IMAGE_BACKUP" ]; then + rollback_agent_image && rolled_back=true + else + echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2 + fi + ;; + daemon,agent) + # Preserve the old all-or-nothing rollback guarantee: never restore one + # component and leave the other on the failed release. + if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then + if rollback_agent_image && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then + rolled_back=true + fi + else + echo "==> not all changed-component backups exist; cannot roll back automatically" >&2 + fi + ;; + *) + echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2 + ;; + esac +else + echo "==> could not stop archied.service; refusing rollback while it may be running" >&2 +fi if [ "$rolled_back" = true ]; then systemctl --user restart archied.service fi