fix(update): make daemon rollback schema-safe (#704)

This commit is contained in:
sam authored and GitHub committed 2026-09-03 14:53:03 +10:00
1 parent c75ed44721
commit bf7115fcb5
5 files changed
+198 -37

No files matched your search

+15 -8
View File
@@ -370,7 +370,6 @@ func (b *boot) setupObservability() {
// doesn't expose it. See docs/prds/payload-field-mapping.md.
if ms, ok := b.st.(store.MappingStore); ok {
b.web.Mappings = ms
b.d.Mappings = ms
} else {
log.Warn("mapping storage unavailable: task store does not implement MappingStore")
}
@@ -383,21 +382,14 @@ func (b *boot) setupObservability() {
// See docs/prds/webhook-intake-security.md.
if bs, ok := b.st.(store.BindingStore); ok {
b.web.Bindings = bs
b.d.Bindings = bs
} else {
log.Warn("binding storage unavailable: task store does not implement BindingStore")
}
if bd, ok := b.st.(store.BindingDispatcher); ok {
b.web.BindingDispatcher = bd
b.d.BindingDispatcher = bd
} else {
log.Warn("binding dispatcher unavailable: task store does not implement BindingDispatcher")
}
if btc, ok := b.st.(store.BindingTaskCreator); ok {
b.d.BindingTaskCreator = btc
} else {
log.Warn("binding task creator unavailable: task store does not implement BindingTaskCreator")
}
sink := bus.Subscribe(256)
go persistAndBroadcastEvents(sink, b.st, b.web, log)
}
@@ -411,6 +403,9 @@ func (b *boot) connectNATS(ctx context.Context) error {
url := cfg.NATS.URL
var natsToken string
if cfg.NATS.Mode == config.NATSModeExternal {
if url == "" {
return fmt.Errorf("nats.url is required when nats.mode is external")
}
var err error
natsToken, err = configuredNATSToken(cfg.NATS, os.Getenv)
if err != nil {
@@ -1242,6 +1237,18 @@ func (b *boot) buildDaemon() {
// this point /api/config and the daemon can never disagree about the
// published config.
b.web.Cfg = b.d.Cfg
if ms, ok := b.st.(store.MappingStore); ok {
b.d.Mappings = ms
}
if bs, ok := b.st.(store.BindingStore); ok {
b.d.Bindings = bs
}
if bd, ok := b.st.(store.BindingDispatcher); ok {
b.d.BindingDispatcher = bd
}
if btc, ok := b.st.(store.BindingTaskCreator); ok {
b.d.BindingTaskCreator = btc
}
b.setupForgeWebhook()
}
@@ -150,3 +150,40 @@ func writeDockerResponse(t *testing.T, w http.ResponseWriter, value any) {
t.Errorf("encode Docker response: %v", err)
}
}
func TestConnectExternalNATSEmptyURLReturnsConfigurationError(t *testing.T) {
b := newBootstrap()
b.cfg.NATS = config.NATSConfig{Mode: config.NATSModeExternal}
err := b.connectNATS(t.Context())
if err == nil || !strings.Contains(err.Error(), "nats.url is required") {
t.Fatalf("connectNATS() error = %v, want useful missing-url error", err)
}
}
func TestSetupObservabilityDoesNotTouchDaemonBeforeBuild(t *testing.T) {
fileset := token.NewFileSet()
file, err := parser.ParseFile(fileset, "bootstrap.go", nil, 0)
if err != nil {
t.Fatal(err)
}
setup := methodBody(t, file, "setupObservability")
var daemonAssignments int
ast.Inspect(setup, func(node ast.Node) bool {
assign, ok := node.(*ast.AssignStmt)
if !ok {
return true
}
for _, lhs := range assign.Lhs {
if selector, ok := lhs.(*ast.SelectorExpr); ok {
if receiver, ok := selector.X.(*ast.SelectorExpr); ok && receiver.Sel.Name == "d" {
daemonAssignments++
}
}
}
return true
})
if daemonAssignments != 0 {
t.Fatalf("setupObservability assigns daemon fields %d times; daemon is built later", daemonAssignments)
}
}
@@ -3,6 +3,7 @@ package releaseupdate
import (
"bufio"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
@@ -121,6 +122,73 @@ func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponent
}
}
func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
reportPath := filepath.Join(work, "report.json")
dbPath, backupPath := filepath.Join(work, "tasks.sqlite"), filepath.Join(work, "tasks.prev.sqlite")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
for name, content := range map[string]string{"archied": "new", "archied.prev": "old"} {
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
for path, version := range map[string]int{dbPath: 2, backupPath: 1} {
if output, err := exec.Command("sqlite3", path, fmt.Sprintf("PRAGMA user_version=%d; CREATE TABLE tasks (id INTEGER);", version)).CombinedOutput(); err != nil {
t.Fatalf("create database %s: %v: %s", path, err, output)
}
}
callsPath := filepath.Join(work, "calls")
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
writeFakeCommand(t, fakeDir, "cp", `printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"; /usr/bin/cp "$@"`)
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
"ARCHIE_TEST_CALLS="+callsPath,
"ARCHIE_UPDATE_REPORT_PATH="+reportPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", "ARCHIE_UPDATE_COMPONENTS=daemon",
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.19.10", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.20.0",
"ARCHIE_TASK_DB_PATH="+dbPath, "ARCHIE_TASK_DB_BACKUP="+backupPath)
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
output, err := exec.Command("sqlite3", dbPath, "PRAGMA user_version;").Output()
if err != nil {
t.Fatal(err)
}
if strings.TrimSpace(string(output)) != "1" {
t.Fatalf("restored schema version = %q, want 1", output)
}
calls, err := os.ReadFile(callsPath)
if err != nil {
t.Fatal(err)
}
callLines := strings.Split(strings.TrimSpace(string(calls)), "\n")
stopIndex, copyIndex, startIndex := -1, -1, -1
for i, line := range callLines {
switch line {
case "systemctl --user stop archied.service":
stopIndex = i
case "systemctl --user restart archied.service":
startIndex = i
}
if strings.HasPrefix(line, "cp ") && strings.Contains(line, "tasks.prev.sqlite") {
copyIndex = i
}
}
if stopIndex < 0 || copyIndex < 0 || stopIndex > copyIndex {
t.Fatalf("rollback must stop service before restoring database; calls = %q", calls)
}
if startIndex < 0 || startIndex <= copyIndex {
t.Fatalf("rollback must restart service after restoring database; calls = %q", calls)
}
}
func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) {
t.Helper()
ctx := t.Context()
+23
View File
@@ -43,9 +43,24 @@ configured_agent_image() {
printf '%s' "$image"
}
configured_task_db_path() {
awk '
/^[[:space:]]*db_path[[:space:]]*=/ {
value = $0
sub(/^[[:space:]]*db_path[[:space:]]*=[[:space:]]*/, "", value)
sub(/[[:space:]]*(#.*)?$/, "", value)
if (value ~ /^".*"$/) { sub(/^"/, "", value); sub(/"$/, "", value); print value }
else if (length(value) >= 2 && substr(value, 1, 1) == sprintf("%c", 39) && substr(value, length(value), 1) == sprintf("%c", 39)) print substr(value, 2, length(value) - 2)
exit
}
' "$CONFIG_PATH" 2>/dev/null || true
}
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-$(configured_agent_image)}"
AGENT_IMAGE="${AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
AGENT_IMAGE_BACKUP="$BIN_DIR/archie-agent-image.prev.tar"
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}"
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}"
PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}"
@@ -124,6 +139,12 @@ install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-updat
install -m755 "$source_dir/scripts/archie-update-watchdog" "$BIN_DIR/archie-update-watchdog"
install -m755 "$source_dir/scripts/archie-update-check" "$BIN_DIR/archie-update-check"
if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_DB_PATH" ]; then
echo "==> backing up task database for schema-safe rollback"
mkdir -p "$(dirname "$TASK_DB_BACKUP")"
sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'"
fi
if [[ ",$components," == *,daemon,* ]]; then
echo "==> building archied $GW"
go build -ldflags "-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=$GW -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=$RT -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" \
@@ -157,6 +178,8 @@ systemd-run --user --unit=archied-update-restart --collect --quiet \
--setenv=ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT" \
--setenv=ARCHIE_AGENT_IMAGE="$AGENT_IMAGE" \
--setenv=ARCHIE_AGENT_IMAGE_BACKUP="$AGENT_IMAGE_BACKUP" \
--setenv=ARCHIE_TASK_DB_PATH="$TASK_DB_PATH" \
--setenv=ARCHIE_TASK_DB_BACKUP="$TASK_DB_BACKUP" \
"$BIN_DIR/archie-update-watchdog"
case "$components" in
+55 -29
View File
@@ -10,6 +10,8 @@ set -uo pipefail
BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}"
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}"
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}"
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
@@ -82,6 +84,20 @@ rollback_agent_image() {
docker image load --input "$AGENT_IMAGE_BACKUP"
}
rollback_task_db() {
if [ -z "$TASK_DB_PATH" ]; then
echo "==> no task database path; cannot restore schema-compatible state" >&2
return 1
fi
if [ ! -f "$TASK_DB_BACKUP" ]; then
echo "==> no task database backup; cannot restore schema-compatible state" >&2
return 1
fi
sqlite3 "$TASK_DB_BACKUP" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true
cp -f "$TASK_DB_BACKUP" "$TASK_DB_PATH"
rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm"
}
echo "==> restarting archied.service"
systemctl --user restart archied.service
@@ -93,36 +109,46 @@ fi
echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components"
rolled_back=false
case "$COMPONENTS" in
daemon)
if [ -f "$BIN_DIR/archied.prev" ]; then
rollback_component archied && rolled_back=true
else
echo "==> no backup for archied; cannot roll it back automatically" >&2
fi
;;
agent)
if [ -f "$AGENT_IMAGE_BACKUP" ]; then
rollback_agent_image && rolled_back=true
else
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
fi
;;
daemon,agent)
# Preserve the old all-or-nothing rollback guarantee: never restore one
# component and leave the other on the failed release.
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ]; then
if rollback_agent_image && rollback_component archied; then
rolled_back=true
# Stop before touching either the executable or its database. The service
# uses Restart=on-failure in the supported unit, so copying rollback state
# while a crash-loop child is alive can race with SQLite and execute the new
# binary after its files have been replaced.
if systemctl --user stop archied.service; then
case "$COMPONENTS" in
daemon)
if [ -f "$BIN_DIR/archied.prev" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then
rolled_back=true
fi
else
echo "==> no backup for archied; cannot roll it back automatically" >&2
fi
else
echo "==> not all changed-component backups exist; cannot roll back automatically" >&2
fi
;;
*)
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
;;
esac
;;
agent)
if [ -f "$AGENT_IMAGE_BACKUP" ]; then
rollback_agent_image && rolled_back=true
else
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
fi
;;
daemon,agent)
# Preserve the old all-or-nothing rollback guarantee: never restore one
# component and leave the other on the failed release.
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
if rollback_agent_image && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then
rolled_back=true
fi
else
echo "==> not all changed-component backups exist; cannot roll back automatically" >&2
fi
;;
*)
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
;;
esac
else
echo "==> could not stop archied.service; refusing rollback while it may be running" >&2
fi
if [ "$rolled_back" = true ]; then
systemctl --user restart archied.service
fi