fix(update): make daemon rollback schema-safe (#704)
This commit is contained in:
1 parent
c75ed44721
commit
bf7115fcb5
5 files changed
+198
-37
No files matched your search
@@ -370,7 +370,6 @@ func (b *boot) setupObservability() {
|
||||
// doesn't expose it. See docs/prds/payload-field-mapping.md.
|
||||
if ms, ok := b.st.(store.MappingStore); ok {
|
||||
b.web.Mappings = ms
|
||||
b.d.Mappings = ms
|
||||
} else {
|
||||
log.Warn("mapping storage unavailable: task store does not implement MappingStore")
|
||||
}
|
||||
@@ -383,21 +382,14 @@ func (b *boot) setupObservability() {
|
||||
// See docs/prds/webhook-intake-security.md.
|
||||
if bs, ok := b.st.(store.BindingStore); ok {
|
||||
b.web.Bindings = bs
|
||||
b.d.Bindings = bs
|
||||
} else {
|
||||
log.Warn("binding storage unavailable: task store does not implement BindingStore")
|
||||
}
|
||||
if bd, ok := b.st.(store.BindingDispatcher); ok {
|
||||
b.web.BindingDispatcher = bd
|
||||
b.d.BindingDispatcher = bd
|
||||
} else {
|
||||
log.Warn("binding dispatcher unavailable: task store does not implement BindingDispatcher")
|
||||
}
|
||||
if btc, ok := b.st.(store.BindingTaskCreator); ok {
|
||||
b.d.BindingTaskCreator = btc
|
||||
} else {
|
||||
log.Warn("binding task creator unavailable: task store does not implement BindingTaskCreator")
|
||||
}
|
||||
sink := bus.Subscribe(256)
|
||||
go persistAndBroadcastEvents(sink, b.st, b.web, log)
|
||||
}
|
||||
@@ -411,6 +403,9 @@ func (b *boot) connectNATS(ctx context.Context) error {
|
||||
url := cfg.NATS.URL
|
||||
var natsToken string
|
||||
if cfg.NATS.Mode == config.NATSModeExternal {
|
||||
if url == "" {
|
||||
return fmt.Errorf("nats.url is required when nats.mode is external")
|
||||
}
|
||||
var err error
|
||||
natsToken, err = configuredNATSToken(cfg.NATS, os.Getenv)
|
||||
if err != nil {
|
||||
@@ -1242,6 +1237,18 @@ func (b *boot) buildDaemon() {
|
||||
// this point /api/config and the daemon can never disagree about the
|
||||
// published config.
|
||||
b.web.Cfg = b.d.Cfg
|
||||
if ms, ok := b.st.(store.MappingStore); ok {
|
||||
b.d.Mappings = ms
|
||||
}
|
||||
if bs, ok := b.st.(store.BindingStore); ok {
|
||||
b.d.Bindings = bs
|
||||
}
|
||||
if bd, ok := b.st.(store.BindingDispatcher); ok {
|
||||
b.d.BindingDispatcher = bd
|
||||
}
|
||||
if btc, ok := b.st.(store.BindingTaskCreator); ok {
|
||||
b.d.BindingTaskCreator = btc
|
||||
}
|
||||
b.setupForgeWebhook()
|
||||
}
|
||||
|
||||
|
||||
@@ -150,3 +150,40 @@ func writeDockerResponse(t *testing.T, w http.ResponseWriter, value any) {
|
||||
t.Errorf("encode Docker response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectExternalNATSEmptyURLReturnsConfigurationError(t *testing.T) {
|
||||
b := newBootstrap()
|
||||
b.cfg.NATS = config.NATSConfig{Mode: config.NATSModeExternal}
|
||||
|
||||
err := b.connectNATS(t.Context())
|
||||
if err == nil || !strings.Contains(err.Error(), "nats.url is required") {
|
||||
t.Fatalf("connectNATS() error = %v, want useful missing-url error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetupObservabilityDoesNotTouchDaemonBeforeBuild(t *testing.T) {
|
||||
fileset := token.NewFileSet()
|
||||
file, err := parser.ParseFile(fileset, "bootstrap.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
setup := methodBody(t, file, "setupObservability")
|
||||
var daemonAssignments int
|
||||
ast.Inspect(setup, func(node ast.Node) bool {
|
||||
assign, ok := node.(*ast.AssignStmt)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
for _, lhs := range assign.Lhs {
|
||||
if selector, ok := lhs.(*ast.SelectorExpr); ok {
|
||||
if receiver, ok := selector.X.(*ast.SelectorExpr); ok && receiver.Sel.Name == "d" {
|
||||
daemonAssignments++
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if daemonAssignments != 0 {
|
||||
t.Fatalf("setupObservability assigns daemon fields %d times; daemon is built later", daemonAssignments)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package releaseupdate
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -121,6 +122,73 @@ func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponent
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) {
|
||||
root, err := filepath.Abs(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
work := t.TempDir()
|
||||
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
|
||||
reportPath := filepath.Join(work, "report.json")
|
||||
dbPath, backupPath := filepath.Join(work, "tasks.sqlite"), filepath.Join(work, "tasks.prev.sqlite")
|
||||
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, content := range map[string]string{"archied": "new", "archied.prev": "old"} {
|
||||
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for path, version := range map[string]int{dbPath: 2, backupPath: 1} {
|
||||
if output, err := exec.Command("sqlite3", path, fmt.Sprintf("PRAGMA user_version=%d; CREATE TABLE tasks (id INTEGER);", version)).CombinedOutput(); err != nil {
|
||||
t.Fatalf("create database %s: %v: %s", path, err, output)
|
||||
}
|
||||
}
|
||||
callsPath := filepath.Join(work, "calls")
|
||||
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
|
||||
writeFakeCommand(t, fakeDir, "cp", `printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"; /usr/bin/cp "$@"`)
|
||||
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
|
||||
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
|
||||
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
|
||||
"ARCHIE_TEST_CALLS="+callsPath,
|
||||
"ARCHIE_UPDATE_REPORT_PATH="+reportPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", "ARCHIE_UPDATE_COMPONENTS=daemon",
|
||||
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.19.10", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.20.0",
|
||||
"ARCHIE_TASK_DB_PATH="+dbPath, "ARCHIE_TASK_DB_BACKUP="+backupPath)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
||||
}
|
||||
output, err := exec.Command("sqlite3", dbPath, "PRAGMA user_version;").Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.TrimSpace(string(output)) != "1" {
|
||||
t.Fatalf("restored schema version = %q, want 1", output)
|
||||
}
|
||||
calls, err := os.ReadFile(callsPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
callLines := strings.Split(strings.TrimSpace(string(calls)), "\n")
|
||||
stopIndex, copyIndex, startIndex := -1, -1, -1
|
||||
for i, line := range callLines {
|
||||
switch line {
|
||||
case "systemctl --user stop archied.service":
|
||||
stopIndex = i
|
||||
case "systemctl --user restart archied.service":
|
||||
startIndex = i
|
||||
}
|
||||
if strings.HasPrefix(line, "cp ") && strings.Contains(line, "tasks.prev.sqlite") {
|
||||
copyIndex = i
|
||||
}
|
||||
}
|
||||
if stopIndex < 0 || copyIndex < 0 || stopIndex > copyIndex {
|
||||
t.Fatalf("rollback must stop service before restoring database; calls = %q", calls)
|
||||
}
|
||||
if startIndex < 0 || startIndex <= copyIndex {
|
||||
t.Fatalf("rollback must restart service after restoring database; calls = %q", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
|
||||
@@ -43,9 +43,24 @@ configured_agent_image() {
|
||||
printf '%s' "$image"
|
||||
}
|
||||
|
||||
configured_task_db_path() {
|
||||
awk '
|
||||
/^[[:space:]]*db_path[[:space:]]*=/ {
|
||||
value = $0
|
||||
sub(/^[[:space:]]*db_path[[:space:]]*=[[:space:]]*/, "", value)
|
||||
sub(/[[:space:]]*(#.*)?$/, "", value)
|
||||
if (value ~ /^".*"$/) { sub(/^"/, "", value); sub(/"$/, "", value); print value }
|
||||
else if (length(value) >= 2 && substr(value, 1, 1) == sprintf("%c", 39) && substr(value, length(value), 1) == sprintf("%c", 39)) print substr(value, 2, length(value) - 2)
|
||||
exit
|
||||
}
|
||||
' "$CONFIG_PATH" 2>/dev/null || true
|
||||
}
|
||||
|
||||
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-$(configured_agent_image)}"
|
||||
AGENT_IMAGE="${AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
|
||||
AGENT_IMAGE_BACKUP="$BIN_DIR/archie-agent-image.prev.tar"
|
||||
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}"
|
||||
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
|
||||
|
||||
PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}"
|
||||
PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}"
|
||||
@@ -124,6 +139,12 @@ install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-updat
|
||||
install -m755 "$source_dir/scripts/archie-update-watchdog" "$BIN_DIR/archie-update-watchdog"
|
||||
install -m755 "$source_dir/scripts/archie-update-check" "$BIN_DIR/archie-update-check"
|
||||
|
||||
if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_DB_PATH" ]; then
|
||||
echo "==> backing up task database for schema-safe rollback"
|
||||
mkdir -p "$(dirname "$TASK_DB_BACKUP")"
|
||||
sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'"
|
||||
fi
|
||||
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
echo "==> building archied $GW"
|
||||
go build -ldflags "-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=$GW -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=$RT -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" \
|
||||
@@ -157,6 +178,8 @@ systemd-run --user --unit=archied-update-restart --collect --quiet \
|
||||
--setenv=ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT" \
|
||||
--setenv=ARCHIE_AGENT_IMAGE="$AGENT_IMAGE" \
|
||||
--setenv=ARCHIE_AGENT_IMAGE_BACKUP="$AGENT_IMAGE_BACKUP" \
|
||||
--setenv=ARCHIE_TASK_DB_PATH="$TASK_DB_PATH" \
|
||||
--setenv=ARCHIE_TASK_DB_BACKUP="$TASK_DB_BACKUP" \
|
||||
"$BIN_DIR/archie-update-watchdog"
|
||||
|
||||
case "$components" in
|
||||
|
||||
@@ -10,6 +10,8 @@ set -uo pipefail
|
||||
BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
|
||||
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
|
||||
AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}"
|
||||
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}"
|
||||
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
|
||||
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}"
|
||||
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
|
||||
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
|
||||
@@ -82,6 +84,20 @@ rollback_agent_image() {
|
||||
docker image load --input "$AGENT_IMAGE_BACKUP"
|
||||
}
|
||||
|
||||
rollback_task_db() {
|
||||
if [ -z "$TASK_DB_PATH" ]; then
|
||||
echo "==> no task database path; cannot restore schema-compatible state" >&2
|
||||
return 1
|
||||
fi
|
||||
if [ ! -f "$TASK_DB_BACKUP" ]; then
|
||||
echo "==> no task database backup; cannot restore schema-compatible state" >&2
|
||||
return 1
|
||||
fi
|
||||
sqlite3 "$TASK_DB_BACKUP" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true
|
||||
cp -f "$TASK_DB_BACKUP" "$TASK_DB_PATH"
|
||||
rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm"
|
||||
}
|
||||
|
||||
echo "==> restarting archied.service"
|
||||
systemctl --user restart archied.service
|
||||
|
||||
@@ -93,36 +109,46 @@ fi
|
||||
|
||||
echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components"
|
||||
rolled_back=false
|
||||
case "$COMPONENTS" in
|
||||
daemon)
|
||||
if [ -f "$BIN_DIR/archied.prev" ]; then
|
||||
rollback_component archied && rolled_back=true
|
||||
else
|
||||
echo "==> no backup for archied; cannot roll it back automatically" >&2
|
||||
fi
|
||||
;;
|
||||
agent)
|
||||
if [ -f "$AGENT_IMAGE_BACKUP" ]; then
|
||||
rollback_agent_image && rolled_back=true
|
||||
else
|
||||
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
|
||||
fi
|
||||
;;
|
||||
daemon,agent)
|
||||
# Preserve the old all-or-nothing rollback guarantee: never restore one
|
||||
# component and leave the other on the failed release.
|
||||
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ]; then
|
||||
if rollback_agent_image && rollback_component archied; then
|
||||
rolled_back=true
|
||||
# Stop before touching either the executable or its database. The service
|
||||
# uses Restart=on-failure in the supported unit, so copying rollback state
|
||||
# while a crash-loop child is alive can race with SQLite and execute the new
|
||||
# binary after its files have been replaced.
|
||||
if systemctl --user stop archied.service; then
|
||||
case "$COMPONENTS" in
|
||||
daemon)
|
||||
if [ -f "$BIN_DIR/archied.prev" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
|
||||
if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then
|
||||
rolled_back=true
|
||||
fi
|
||||
else
|
||||
echo "==> no backup for archied; cannot roll it back automatically" >&2
|
||||
fi
|
||||
else
|
||||
echo "==> not all changed-component backups exist; cannot roll back automatically" >&2
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
agent)
|
||||
if [ -f "$AGENT_IMAGE_BACKUP" ]; then
|
||||
rollback_agent_image && rolled_back=true
|
||||
else
|
||||
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
|
||||
fi
|
||||
;;
|
||||
daemon,agent)
|
||||
# Preserve the old all-or-nothing rollback guarantee: never restore one
|
||||
# component and leave the other on the failed release.
|
||||
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
|
||||
if rollback_agent_image && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then
|
||||
rolled_back=true
|
||||
fi
|
||||
else
|
||||
echo "==> not all changed-component backups exist; cannot roll back automatically" >&2
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
|
||||
;;
|
||||
esac
|
||||
else
|
||||
echo "==> could not stop archied.service; refusing rollback while it may be running" >&2
|
||||
fi
|
||||
if [ "$rolled_back" = true ]; then
|
||||
systemctl --user restart archied.service
|
||||
fi
|
||||
|
||||
Reference in new issue
Block a user