diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 65dbc3dee..e8eb897c1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -166,6 +166,8 @@ jobs: fi build-and-push: + outputs: + agent_digest: ${{ steps.agent.outputs.digest }} needs: gate runs-on: ubuntu-latest steps: @@ -258,6 +260,7 @@ jobs: cache-to: type=gha,mode=max,scope=archied - name: Build and push archie-agent + id: agent if: steps.versions.outputs.runtime_tag != '' || steps.versions.outputs.release_tag != '' uses: docker/build-push-action@v6 with: @@ -282,7 +285,7 @@ jobs: # the Docker image build-and-push already publishes, never as a host # binary (archie-core-8cda.4.7.1). dist-zip: - needs: gate + needs: [gate, build-and-push] # Not gated on github.ref. A release is pushed as main plus both tags at # one SHA, the concurrency group below collapses those into a single run, # and the survivor is whichever ref happened to start last -- for @@ -329,7 +332,7 @@ jobs: # list and the copy list below stop matching the commands in cmd/. ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=${{ steps.version.outputs.version }} -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=${{ steps.version.outputs.version }}" mkdir -p dist - for cmd in archied archie-gateway archie-state-store archie-ui archie-messaging; do + for cmd in $(jq -r '.required_topology.units[]' release.json); do go build -ldflags "$ldflags" -o "dist/$cmd" "./cmd/$cmd" done @@ -339,19 +342,28 @@ jobs: set -e name="archie-core-archied-v${{ steps.version.outputs.version }}-linux-amd64" mkdir -p "$name" - cp dist/archied dist/archie-gateway dist/archie-state-store dist/archie-ui dist/archie-messaging "$name/" + for cmd in $(jq -r '.required_topology.units[]' release.json); do + cp "dist/$cmd" "$name/" + done cp deployments/INSTRUCTIONS.md CHANGELOG.md "$name/" + cp scripts/archie-update-* "$name/" + digest='${{ needs.build-and-push.outputs.agent_digest }}' + test -n "$digest" + jq --arg image "ghcr.io/samcharles93/archie-agent@$digest" '. + {agent_image: $image}' release.json > "$name/release.json" # README.md doesn't exist yet in this repo; include it once it does # rather than fail the release on its absence. [ -f README.md ] && cp README.md "$name/" zip -r "$name.zip" "$name" + sha256sum "$name.zip" > SHA256SUMS echo "name=$name" >> "$GITHUB_OUTPUT" - name: Upload distribution zip uses: actions/upload-artifact@v4 with: name: ${{ steps.zip.outputs.name }} - path: ${{ steps.zip.outputs.name }}.zip + path: | + ${{ steps.zip.outputs.name }}.zip + SHA256SUMS retention-days: 90 # A Release is what an operator can actually download from; the @@ -370,7 +382,10 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e - tag="archied/v${{ steps.version.outputs.version }}" + tag="v${{ steps.version.outputs.version }}" + if ! git ls-remote --exit-code --tags origin "refs/tags/$tag" >/dev/null; then + tag="archied/v${{ steps.version.outputs.version }}" + fi version="${{ steps.version.outputs.version }}" # The release body is this version's changelog section, not the whole # file: from its "## [x.y.z]" heading to the next "## [" heading. @@ -391,11 +406,11 @@ jobs: # alone, so a re-run of this job (a retried or superseded deploy) # cannot fail on an existing Release or duplicate the zip. if gh release view "$tag" >/dev/null 2>&1; then - gh release upload "$tag" "${{ steps.zip.outputs.name }}.zip" --clobber + gh release upload "$tag" "${{ steps.zip.outputs.name }}.zip" SHA256SUMS --clobber else gh release create "$tag" \ --title "archied v$version" \ --notes-file "$notes" \ - "${{ steps.zip.outputs.name }}.zip" + "${{ steps.zip.outputs.name }}.zip" SHA256SUMS fi echo "published $tag with ${{ steps.zip.outputs.name }}.zip" diff --git a/cmd/archied/main.go b/cmd/archied/main.go index bdc0314a0..cee2db186 100644 --- a/cmd/archied/main.go +++ b/cmd/archied/main.go @@ -16,6 +16,15 @@ func main() { if archied.IsSetupArgs(args) { os.Exit(archied.RunSetup(args[1:], os.Stdin, os.Stdout, os.Stderr)) } + if len(args) > 0 && args[0] == "select-release" { + os.Exit(archied.RunReleaseChannel(args[1:], os.Stdin, os.Stdout, os.Stderr)) + } + if len(args) > 0 && args[0] == "update-image" { + os.Exit(archied.RunUpdateImage(args[1:], os.Stderr)) + } + if len(args) > 0 && args[0] == "update-topology" { + os.Exit(archied.RunUpdateTopology(args[1:], os.Stdout, os.Stderr)) + } if len(args) > 0 && args[0] == "status" { os.Exit(archied.RunStatus(args[1:], os.Stdout, os.Stderr)) } diff --git a/deployments/INSTRUCTIONS.md b/deployments/INSTRUCTIONS.md index 623824d60..285a078bc 100644 --- a/deployments/INSTRUCTIONS.md +++ b/deployments/INSTRUCTIONS.md @@ -86,3 +86,45 @@ only `archied` and `archie-agent` are independently versioned). carries the Telegram, email and webhook channels since the v1.30.0 extraction, so an install (or an upgrade across v1.30.0) needs its unit as well -- see `deployments/systemd-user-service.md`. + +### Native self-update + +`archie-update-install` downloads the approved release's Linux/amd64 archive +and verifies its `SHA256SUMS` entry before executing or installing it. Releases +must include `release.json`, all host binaries, and the three updater scripts. +Older releases without this contract cannot be installed through the artifact +path. `--from-source` (or `ARCHIE_UPDATE_SOURCE_BUILD=1`) explicitly clones and +builds the selected release instead. + +The update check command accepts `--channel stable` (default), `--channel next` +(includes prereleases), or `--channel exact-pin --pin 1.2.3`. Set these arguments +in the control plane's channel settings alongside the existing check command. +The notification destination remains separate from the release channel. + +For scheduled, unattended runs, `archie-update-install --auto` discovers the +selected release and refuses if the daemon reports work in flight. It checks +again immediately before promotion. A scheduler can set +`ARCHIE_UPDATE_RELEASE_CHANNEL` and `ARCHIE_UPDATE_PIN`; nothing schedules updates +by default. All services stop through their ordinary graceful shutdown path. + +`release.json` supplies the service order and required config sections to both +first-install and self-update. A missing unit or config section produces an +ordered migration plan and refuses installation. Apply that plan before retrying. +Automatic topology edits are intentionally unsupported. + +The watchdog switches staged executables by rename, restarts services, and +requires the candidate daemon's existing readiness registry to remain healthy +for 120 seconds. Agent images come from the release manifest by digest. The +image switch updates the control-plane policy, preserving other settings; +rollback restores the prior image policy with conflict detection. Source builds +use the built image's immutable image ID. + +Transactions live in `${XDG_STATE_HOME:-~/.local/state}/archie/update`, outside +the binary directory. `phase`, `environment`, `image.json` (when applicable), +and `previous/` describe the transaction. A lock reports the owning process. +On daemon boot, an interrupted transaction schedules independent recovery before +normal startup. To recover manually, run `transaction/watchdog --recover` from +that state directory. A failed recovery keeps the journal for another attempt. + +Database snapshots remain available for manual recovery. Schema migrations and +PostgreSQL data are never automatically reversed. diff --git a/go.mod b/go.mod index b06c56e7c..04b6ba530 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,6 @@ require ( golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect golang.org/x/sync v0.23.0 // indirect - golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 // indirect @@ -119,8 +118,10 @@ require ( go.lsp.dev/jsonrpc2 v1.0.1 go.lsp.dev/protocol v1.0.1 go.lsp.dev/uri v1.0.1 + golang.org/x/mod v0.41.0 golang.org/x/net v0.59.0 golang.org/x/oauth2 v0.37.0 + golang.org/x/sys v0.48.0 golang.org/x/term v0.46.0 google.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e google.golang.org/protobuf v1.36.12 diff --git a/go.sum b/go.sum index dd364246c..fd1feac4f 100644 --- a/go.sum +++ b/go.sum @@ -270,6 +270,8 @@ golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU= golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= diff --git a/install.sh b/install.sh index 7817342bf..a6e63c640 100755 --- a/install.sh +++ b/install.sh @@ -112,7 +112,7 @@ if [ "$(uname -s)" != "Linux" ]; then exit 1 fi -for cmd in git go; do +for cmd in git go jq; do if ! command -v "$cmd" &>/dev/null; then echo "Error: '$cmd' is required but not installed." >&2 exit 1 @@ -159,6 +159,8 @@ else fi fi +TOPOLOGY_UNITS="$(jq -er '.required_topology.units | join(" ")' "$SRC_DIR/release.json")" + # 4. Build and install the native daemon. archie-agent is deployed only as # the managed task image; installing a host binary would imply an unsupported # host execution path. @@ -193,10 +195,10 @@ echo "==> Building native archie binaries..." # webhook channels dead with no error anywhere. This list, # the zip's two lists and the two in scripts/archie-update-install must agree; # TestDistZipShipsEveryHostCommand fails when they do not. - for cmd in archied archie-gateway archie-state-store archie-ui archie-messaging; do + for cmd in $TOPOLOGY_UNITS; do go build -ldflags "${LDFLAGS}" -o "${ARCHIE_BIN_DIR}/${cmd}" "./cmd/${cmd}" done - install -m755 "${SRC_DIR}/scripts/archie-update-install" "${ARCHIE_BIN_DIR}/archie-update-install" + install -m755 "${SRC_DIR}"/scripts/archie-update-* "${ARCHIE_BIN_DIR}/" ) echo " Installed archied, archie-gateway, archie-state-store, archie-ui, archie-messaging and updater to ${ARCHIE_BIN_DIR}/" @@ -520,7 +522,8 @@ EOF systemctl --user daemon-reload || true SERVICE_INSTALLED=true - for unit in archied archie-state-store archie-gateway archie-ui archie-messaging; do + for unit in $TOPOLOGY_UNITS; do + test -f "${SYSTEMD_USER_DIR}/${unit}.service" || { echo "missing required unit: $unit" >&2; exit 1; } echo " Installed ${SYSTEMD_USER_DIR}/${unit}.service" done @@ -546,7 +549,7 @@ EOF # not race a dial it depends on. if [ "${AUTO_START}" = true ]; then echo "==> Enabling and starting Archie services..." - systemctl --user enable --now archie-state-store archie-gateway archied archie-ui archie-messaging || + systemctl --user enable --now $TOPOLOGY_UNITS || echo " Notice: Could not start every service automatically. Check: systemctl --user status archied archie-state-store archie-gateway archie-ui archie-messaging" fi fi diff --git a/internal/app/archied/health.go b/internal/app/archied/health.go index b30a14c86..d896fb79f 100644 --- a/internal/app/archied/health.go +++ b/internal/app/archied/health.go @@ -7,6 +7,7 @@ import ( "sync/atomic" "github.com/samcharles93/archie-core/internal/app/servicekit" + "github.com/samcharles93/archie-core/internal/buildinfo" "github.com/samcharles93/archie-core/internal/domain/health" ) @@ -24,6 +25,7 @@ import ( type healthSurface struct { serving atomic.Bool registry func() *health.Registry + inFlight func() int } // markServing tolerates a nil surface so a boot path that serves no health @@ -70,7 +72,15 @@ func (h *healthSurface) handleDetailed(w http.ResponseWriter, r *http.Request) { if report.Status != health.StatusOK { w.WriteHeader(http.StatusServiceUnavailable) } - _ = json.NewEncoder(w).Encode(report) + inFlight := -1 + if h.serving.Load() && h.inFlight != nil { + inFlight = h.inFlight() + } + _ = json.NewEncoder(w).Encode(struct { + health.Report + Version string `json:"version"` + InFlight int `json:"in_flight"` + }{Report: report, Version: buildinfo.Version, InFlight: inFlight}) } // startHealth binds the daemon's liveness surface on [health].listen. A @@ -78,7 +88,15 @@ func (h *healthSurface) handleDetailed(w http.ResponseWriter, r *http.Request) { // updates verified, and the watchdog would roll back every release it // installs. func (b *boot) startHealth(ctx context.Context) error { - b.health = &healthSurface{registry: func() *health.Registry { return b.healthRegistry }} + b.health = &healthSurface{ + registry: func() *health.Registry { return b.healthRegistry }, + inFlight: func() int { + if b.d == nil { + return -1 + } + return b.d.InFlight() + }, + } addr := b.cfg.Health.Listen if err := b.serveHealth(ctx, addr, b.health.handler(), "daemon health"); err != nil { b.log.Error("health listener failed", "addr", addr, "err", err) diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 91cbdcdd6..75e7a601f 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -123,6 +123,13 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence return 0 } + if recovering, err := resumeUpdate(context.Background()); err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } else if recovering { + return 0 + } + root, cancelRoot := context.WithCancel(context.Background()) ctx, stop := signal.NotifyContext(root, os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/internal/app/archied/release_channel.go b/internal/app/archied/release_channel.go new file mode 100644 index 000000000..4bc31f915 --- /dev/null +++ b/internal/app/archied/release_channel.go @@ -0,0 +1,32 @@ +package archied + +import ( + "flag" + "fmt" + "io" + "strings" + + "github.com/samcharles93/archie-core/internal/releaseupdate" +) + +func RunReleaseChannel(args []string, in io.Reader, out, stderr io.Writer) int { + flags := flag.NewFlagSet("select-release", flag.ContinueOnError) + flags.SetOutput(stderr) + channel := flags.String("channel", "stable", "stable, next, or exact-pin") + pin := flags.String("pin", "", "exact release version") + if err := flags.Parse(args); err != nil { + return 1 + } + data, err := io.ReadAll(in) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + version, err := releaseupdate.SelectRelease(strings.Fields(string(data)), *channel, *pin) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + fmt.Fprintln(out, version) + return 0 +} diff --git a/internal/app/archied/update.go b/internal/app/archied/update.go new file mode 100644 index 000000000..2f5e2b201 --- /dev/null +++ b/internal/app/archied/update.go @@ -0,0 +1,150 @@ +package archied + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/BurntSushi/toml" + "golang.org/x/sys/unix" + + "github.com/samcharles93/archie-core/internal/releaseupdate" +) + +// RunUpdateTopology observes the host using the candidate's topology contract. +func RunUpdateTopology(args []string, out, stderr io.Writer) int { + flags := flag.NewFlagSet("update-topology", flag.ContinueOnError) + flags.SetOutput(stderr) + configPath := flags.String("config", "", "bootstrap config file") + releasePath := flags.String("release", "", "release manifest") + if err := flags.Parse(args); err != nil { + return 1 + } + if err := checkUpdateTopology(*configPath, *releasePath, out); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + return 0 +} + +func checkUpdateTopology(configPath, releasePath string, out io.Writer) error { + data, err := os.ReadFile(releasePath) + if err != nil { + return err + } + var manifest struct { + Required releaseupdate.Topology `json:"required_topology"` + } + if err := json.Unmarshal(data, &manifest); err != nil { + return err + } + if len(manifest.Required.Units) == 0 { + return errors.New("release has no required_topology") + } + for _, unit := range manifest.Required.Units { + if !regexp.MustCompile(`^archie[a-z0-9-]*$`).MatchString(unit) { + return fmt.Errorf("invalid release unit %q", unit) + } + } + data, err = os.ReadFile(configPath) + if err != nil { + return err + } + var document map[string]any + if err := toml.Unmarshal(data, &document); err != nil { + return fmt.Errorf("observe topology: %w", err) + } + sections := make(map[string]bool) + for _, section := range manifest.Required.ConfigSections { + sections[section] = hasConfigSection(document, section) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + output, observeErr := exec.CommandContext(ctx, "systemctl", "--user", "list-unit-files", "archie*.service", "--no-legend", "--no-pager").Output() + units := observedUnits(output, observeErr) + plan := manifest.Required.MigrationPlan(units, sections) + if len(plan) != 0 { + fmt.Fprintln(out, "Required migration, in order:") + for _, step := range plan { + fmt.Fprintln(out, " -", step) + } + if observeErr != nil { + return fmt.Errorf("topology unknown; apply the printed migration before retrying: %w", observeErr) + } + return errors.New("refusing partial installation; apply the printed migration before retrying") + } + return nil +} + +func hasConfigSection(document map[string]any, section string) bool { + for part := range strings.SplitSeq(section, ".") { + next, ok := document[part].(map[string]any) + if !ok { + return false + } + document = next + } + return true +} + +func updateStateDir() string { + if dir := os.Getenv("ARCHIE_UPDATE_STATE_DIR"); dir != "" { + return dir + } + dir := os.Getenv("XDG_STATE_HOME") + if dir == "" { + home, _ := os.UserHomeDir() + dir = filepath.Join(home, ".local", "state") + } + return filepath.Join(dir, "archie", "update") +} + +// Recovery runs independently of the daemon unit: it must be able to stop that +// unit and replace its executable without systemd killing the recovery process. +func resumeUpdate(ctx context.Context) (bool, error) { + journal := filepath.Join(updateStateDir(), "transaction") + if _, err := os.Stat(journal); errors.Is(err, os.ErrNotExist) { + return false, nil + } else if err != nil { + return false, err + } + lock, err := os.OpenFile(filepath.Join(updateStateDir(), "lock"), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return false, err + } + defer lock.Close() + if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB); errors.Is(err, unix.EWOULDBLOCK) { + return false, nil + } else if err != nil { + return false, err + } + defer func() { _ = unix.Flock(int(lock.Fd()), unix.LOCK_UN) }() + cmd := exec.CommandContext(ctx, "systemd-run", "--user", "--collect", "--quiet", "--unit=archied-update-recovery", "--setenv=ARCHIE_UPDATE_STATE_DIR="+updateStateDir(), filepath.Join(journal, "watchdog"), "--boot-recover") + if output, err := cmd.CombinedOutput(); err != nil { + return false, fmt.Errorf("schedule update recovery: %w: %s", err, output) + } + return true, nil +} + +func observedUnits(output []byte, err error) map[string]bool { + units := make(map[string]bool) + if err != nil { + return units + } + for line := range strings.SplitSeq(string(output), "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[1] != "masked" { + units[strings.TrimSuffix(fields[0], ".service")] = true + } + } + return units +} diff --git a/internal/app/archied/update_image.go b/internal/app/archied/update_image.go new file mode 100644 index 000000000..09d47ca92 --- /dev/null +++ b/internal/app/archied/update_image.go @@ -0,0 +1,117 @@ +package archied + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "regexp" + "time" + + "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/app/servicekit" + pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/domain/presence" + "github.com/samcharles93/archie-core/internal/secret" +) + +type imageChange struct { + Previous string `json:"previous"` + Candidate string `json:"candidate"` +} + +// RunUpdateImage journals and switches the control-plane image policy. It only +// changes the image field, preserving concurrent edits to unrelated settings. +func RunUpdateImage(args []string, stderr io.Writer) int { + flags := flag.NewFlagSet("update-image", flag.ContinueOnError) + flags.SetOutput(stderr) + configPath := flags.String("config", "", "bootstrap config") + journal := flags.String("journal", "", "transaction directory") + candidate := flags.String("prepare", "", "candidate image digest") + rollback := flags.Bool("rollback", false, "restore the prior image") + if err := flags.Parse(args); err != nil { + return 1 + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := updateImage(ctx, *configPath, *journal, *candidate, *rollback); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + return 0 +} + +func updateImage(ctx context.Context, configPath, journal, candidate string, rollback bool) error { + _, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), configPath, "") + if err != nil { + return err + } + client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, doc.Config.Services, secret.NewRegistry()) + if err != nil { + return err + } + defer cleanup() + rpc := client.ControlPlane() + response, err := rpc.Query(ctx, &pb.QueryRequest{Kind: controlplane.ContainerRuntimePoliciesKind}) + if err != nil { + return err + } + var policy map[string]json.RawMessage + if err := json.Unmarshal(response.Resource.ValueJson, &policy); err != nil { + return err + } + var current string + if err := json.Unmarshal(policy["image"], ¤t); err != nil { + return err + } + path := filepath.Join(journal, "image.json") + if candidate != "" { + return prepareImageChange(path, current, candidate) + } + + data, err := os.ReadFile(path) + if err != nil { + return err + } + var change imageChange + if err := json.Unmarshal(data, &change); err != nil { + return err + } + expected, desired := change.Previous, change.Candidate + if rollback { + expected, desired = desired, expected + } + if current == desired { + return nil + } + if current != expected { + return errors.New("image policy changed during update; refusing to overwrite it") + } + policy["image"], err = json.Marshal(desired) + if err != nil { + return err + } + data, err = json.Marshal(policy) + if err != nil { + return err + } + _, err = rpc.Command(ctx, &pb.CommandRequest{Kind: controlplane.ContainerRuntimePoliciesKind, Command: "replace", ValueJson: data, ExpectedVersion: response.Resource.Version, Actor: "self-update", Source: "installer", RequestId: fmt.Sprintf("self-update-%x-%d", sha256.Sum256(data), response.Resource.Version)}) + return err +} + +func prepareImageChange(path, current, candidate string) error { + if !regexp.MustCompile(`^([a-zA-Z0-9./:_-]+@)?sha256:[a-f0-9]{64}$`).MatchString(candidate) { + return errors.New("candidate image must be pinned by digest") + } + data, err := json.Marshal(imageChange{Previous: current, Candidate: candidate}) + if err != nil { + return err + } + return os.WriteFile(path, data, 0o600) +} diff --git a/internal/app/archied/update_test.go b/internal/app/archied/update_test.go new file mode 100644 index 000000000..54dce09b2 --- /dev/null +++ b/internal/app/archied/update_test.go @@ -0,0 +1,75 @@ +package archied + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/sys/unix" +) + +func TestUpdateTopologyRefusal(t *testing.T) { + root := t.TempDir() + manifest := filepath.Join(root, "release.json") + config := filepath.Join(root, "config.toml") + writeUpdateFixture(t, manifest, `{"required_topology":{"units":["archie-state-store","archie-gateway","archied"],"config_sections":["services.state","services.gateway"]}}`) + writeUpdateFixture(t, config, "[services.state]\ntarget='localhost:9090'\n") + writeUpdateFixture(t, filepath.Join(root, "systemctl"), "#!/bin/sh\nprintf 'archied.service enabled enabled\\n'\n") + t.Setenv("PATH", root+":"+os.Getenv("PATH")) + var output bytes.Buffer + if err := checkUpdateTopology(config, manifest, &output); err == nil { + t.Fatal("partial topology accepted") + } + for _, missing := range []string{"configure [services.gateway]", "install and enable archie-state-store.service", "install and enable archie-gateway.service"} { + if !strings.Contains(output.String(), missing) { + t.Fatalf("missing step %q: %s", missing, output.String()) + } + } +} + +func TestResumeUpdate(t *testing.T) { + for _, active := range []bool{false, true} { + t.Run(map[bool]string{false: "interrupted", true: "live"}[active], func(t *testing.T) { + root := t.TempDir() + t.Setenv("ARCHIE_UPDATE_STATE_DIR", root) + if err := os.Mkdir(filepath.Join(root, "transaction"), 0o700); err != nil { + t.Fatal(err) + } + called := filepath.Join(root, "called") + writeUpdateFixture(t, filepath.Join(root, "systemd-run"), "#!/bin/sh\nprintf '%s\\n' \"$@\" > '"+called+"'\n") + t.Setenv("PATH", root+":"+os.Getenv("PATH")) + lock, err := os.OpenFile(filepath.Join(root, "lock"), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + if active { + if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil { + t.Fatal(err) + } + } + resumed, err := resumeUpdate(t.Context()) + if err != nil || resumed == active { + t.Fatalf("resumed=%v err=%v", resumed, err) + } + if !active { + data, err := os.ReadFile(called) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "--boot-recover") { + t.Fatalf("not a rollback: %s", data) + } + } + }) + } +} + +func writeUpdateFixture(t *testing.T, path, data string) { + t.Helper() + if err := os.WriteFile(path, []byte(data), 0o700); err != nil { + t.Fatal(err) + } +} diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 12a8ba428..d3f19b5f4 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -944,7 +944,7 @@ type taskDispatcher struct { active int slotsFree *sync.Cond repoTail map[string]chan struct{} - wg sync.WaitGroup + pending int } // SetMaxConcurrency accepts a new global limit without a restart. Running @@ -1004,7 +1004,16 @@ func (d *taskDispatcher) Submit( d.mu.Unlock() } - d.wg.Go(func() { + d.mu.Lock() + d.pending++ + d.mu.Unlock() + go func() { + defer func() { + d.mu.Lock() + d.pending-- + d.mu.Unlock() + d.slotsFree.Broadcast() + }() if previous != nil { <-previous } @@ -1021,11 +1030,15 @@ func (d *taskDispatcher) Submit( }() } process(ctx, task) - }) + }() } func (d *taskDispatcher) Wait() { - d.wg.Wait() + d.mu.Lock() + defer d.mu.Unlock() + for d.pending > 0 { + d.slotsFree.Wait() + } } func (d *Daemon) poll(ctx context.Context) { @@ -2214,3 +2227,12 @@ func (d *Daemon) RemoveWorktree(owner, repo, identity string, issue int) error { } return trees.Cleanup(owner, repo, issue) } + +// InFlight counts the same submitted work that graceful shutdown waits for, +// including tasks waiting for a repository or concurrency slot. +func (d *Daemon) InFlight() int { + dispatcher := d.taskDispatcher() + dispatcher.mu.Lock() + defer dispatcher.mu.Unlock() + return dispatcher.pending +} diff --git a/internal/daemon/update_test.go b/internal/daemon/update_test.go new file mode 100644 index 000000000..d5cf7fc70 --- /dev/null +++ b/internal/daemon/update_test.go @@ -0,0 +1,39 @@ +package daemon + +import ( + "context" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/domain/workflow" +) + +func TestDispatcherWaitIncludesQueuedWork(t *testing.T) { + dispatcher := newTaskDispatcher(1, nil) + started := make(chan struct{}, 2) + release := make(chan struct{}) + process := func() { started <- struct{}{}; <-release } + for range 2 { + dispatcher.Submit(t.Context(), &workflow.Task{Owner: "o", Repo: "r"}, func(_ context.Context, _ *workflow.Task) { process() }) + } + <-started + dispatcher.mu.Lock() + count := dispatcher.pending + dispatcher.mu.Unlock() + if count != 2 { + t.Fatalf("in-flight count=%d, want running and queued task", count) + } + done := make(chan struct{}) + go func() { dispatcher.Wait(); close(done) }() + select { + case <-done: + t.Fatal("drain completed with pending work") + default: + } + close(release) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("drain did not finish") + } +} diff --git a/internal/releaseupdate/channel.go b/internal/releaseupdate/channel.go new file mode 100644 index 000000000..e52fd268a --- /dev/null +++ b/internal/releaseupdate/channel.go @@ -0,0 +1,54 @@ +package releaseupdate + +import ( + "fmt" + "regexp" + "strings" + + "golang.org/x/mod/semver" +) + +// SelectRelease compares release versions, never git-describe strings. Exact +// pins still have to exist in the catalog; a pin cannot invent a release. +func SelectRelease(tags []string, channel, pin string) (string, error) { + if channel == "" { + channel = "stable" + } + if channel != "stable" && channel != "next" && channel != "exact-pin" { + return "", fmt.Errorf("unknown release channel %q", channel) + } + pin = strings.TrimPrefix(pin, "v") + if channel == "exact-pin" && !semver.IsValid("v"+pin) { + return "", fmt.Errorf("exact-pin requires a semantic version") + } + fullVersion := regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+([+-].*)?$`) + newest := "" + for _, tag := range tags { + tag = strings.TrimPrefix(tag, "refs/tags/") + tag = strings.TrimPrefix(tag, "archied/") + if !fullVersion.MatchString(tag) || !semver.IsValid(tag) { + continue + } + if !releaseMatchesChannel(tag, channel, pin) { + continue + } + if semver.Compare(tag, newest) > 0 { + newest = tag + } + } + if newest == "" { + return "", fmt.Errorf("no release found for channel %s (pin %s)", channel, pin) + } + return strings.TrimPrefix(newest, "v"), nil +} + +func releaseMatchesChannel(tag, channel, pin string) bool { + switch channel { + case "stable": + return semver.Prerelease(tag) == "" + case "exact-pin": + return tag == "v"+pin + default: + return true + } +} diff --git a/internal/releaseupdate/channel_test.go b/internal/releaseupdate/channel_test.go new file mode 100644 index 000000000..025732d85 --- /dev/null +++ b/internal/releaseupdate/channel_test.go @@ -0,0 +1,24 @@ +package releaseupdate + +import "testing" + +func TestSelectRelease(t *testing.T) { + tags := []string{"archied/v1.9.0", "v1.10.0", "archied/v2.0.0-rc.2", "v2.0.0-rc.10", "archie/v9.0.0", "nonsense"} + for _, tc := range []struct{ channel, pin, want string }{ + {"stable", "", "1.10.0"}, + {"", "", "1.10.0"}, + {"next", "", "2.0.0-rc.10"}, + {"exact-pin", "1.9.0", "1.9.0"}, + {"exact-pin", "2.0.0-rc.2", "2.0.0-rc.2"}, + {"exact-pin", "9.0.0", ""}, + {"exact-pin", "", ""}, + {"invalid", "", ""}, + } { + t.Run(tc.channel+tc.pin, func(t *testing.T) { + got, err := SelectRelease(tags, tc.channel, tc.pin) + if got != tc.want || (err != nil) != (tc.want == "") { + t.Fatalf("got %q, %v; want %q", got, err, tc.want) + } + }) + } +} diff --git a/internal/releaseupdate/command.go b/internal/releaseupdate/command.go index b720df6d9..f60da1b4b 100644 --- a/internal/releaseupdate/command.go +++ b/internal/releaseupdate/command.go @@ -19,13 +19,21 @@ const updateResultSentinel = "ARCHIE_UPDATE_RESULT " // CommandCatalog reads a Snapshot JSON document from an explicitly configured // argv command. It never invokes a shell; deployment tooling remains an // administrator-owned adapter. -type CommandCatalog struct{ Command []string } +type CommandCatalog struct { + Command []string + ReleaseChannel string + Pin string +} func (c CommandCatalog) Check(ctx context.Context) (Snapshot, error) { if len(c.Command) == 0 { return Snapshot{}, fmt.Errorf("update check command is empty") } - output, err := exec.CommandContext(ctx, c.Command[0], c.Command[1:]...).Output() + cmd := exec.CommandContext(ctx, c.Command[0], c.Command[1:]...) + if c.ReleaseChannel != "" { + cmd.Env = append(cmd.Environ(), "ARCHIE_UPDATE_RELEASE_CHANNEL="+c.ReleaseChannel, "ARCHIE_UPDATE_PIN="+c.Pin) + } + output, err := cmd.Output() if err != nil { return Snapshot{}, fmt.Errorf("run update check: %w", err) } diff --git a/internal/releaseupdate/installer_test.go b/internal/releaseupdate/installer_test.go new file mode 100644 index 000000000..861471f0e --- /dev/null +++ b/internal/releaseupdate/installer_test.go @@ -0,0 +1,215 @@ +package releaseupdate + +import ( + "archive/zip" + "crypto/sha256" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "golang.org/x/sys/unix" +) + +func TestUpdateTransaction(t *testing.T) { + for _, scenario := range []string{"healthy", "checksum", "unsafe-path", "missing-digest", "busy", "lock", "unhealthy", "crash"} { + t.Run(scenario, func(t *testing.T) { + f := newUpdateFixture(t, scenario) + var lock *os.File + if scenario == "lock" { + var err error + lock, err = os.OpenFile(filepath.Join(f.state, "lock"), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil { + t.Fatal(err) + } + fixtureWrite(t, filepath.Join(f.state, "holder"), "fixture-owner", 0o600) + } + args := []string{} + if scenario == "busy" { + args = append(args, "--auto") + } + output, err := f.run(t, f.installer, args...) + refusal := map[string]string{"checksum": "checksum mismatch", "unsafe-path": "unsafe archive path", "missing-digest": "digest-pinned", "busy": "work in flight", "lock": "fixture-owner"}[scenario] + if refusal != "" { + if err == nil || !strings.Contains(output, refusal) { + t.Fatalf("wanted %q refusal; err=%v output=%s", refusal, err, output) + } + f.assertVersion(t, "old") + return + } + if err != nil { + t.Fatalf("installer: %v\n%s", err, output) + } + f.assertVersion(t, "old") // Staging cannot mutate a running executable. + old, err := os.Open(filepath.Join(f.bin, "archied")) + if err != nil { + t.Fatal(err) + } + defer old.Close() + watchdog := filepath.Join(f.state, "transaction", "watchdog") + output, err = f.run(t, watchdog) + if scenario == "crash" { + if err == nil { + t.Fatal("expected injected crash") + } + f.env = append(f.env, "CRASH_AFTER=") + output, err = f.run(t, watchdog, "--boot-recover") + } + if err != nil { + t.Fatalf("watchdog: %v\n%s", err, output) + } + want := "old" + if scenario == "healthy" { + want = "new" + } + f.assertVersion(t, want) + data := make([]byte, 100) + n, _ := old.Read(data) + if !strings.Contains(string(data[:n]), "old") { + t.Fatal("promotion overwrote the open old inode") + } + if _, err := os.Stat(filepath.Join(f.state, "transaction")); !os.IsNotExist(err) { + t.Fatalf("journal not retired: %v", err) + } + }) + } +} + +type updateFixture struct { + bin, state, installer string + env []string +} + +func fixtureWrite(t *testing.T, path, data string, mode os.FileMode) { + t.Helper() + if err := os.WriteFile(path, []byte(data), mode); err != nil { + t.Fatal(err) + } +} + +func newUpdateFixture(t *testing.T, scenario string) updateFixture { + t.Helper() + root := t.TempDir() + bin, state, fake := filepath.Join(root, "bin"), filepath.Join(root, "state"), filepath.Join(root, "fake") + for _, dir := range []string{bin, state, fake} { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + } + scripts, err := filepath.Abs("../../scripts") + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"archied", "archie-state-store", "archie-gateway", "archie-ui", "archie-messaging"} { + fixtureWrite(t, filepath.Join(bin, name), "#!/bin/bash\n# old\nexit 0\n", 0o755) + } + config := filepath.Join(root, "config.toml") + fixtureWrite(t, config, "[services.state]\n[services.gateway]\n", 0o600) + archive := makeUpdateArchive(t, root, scripts, scenario) + sum := sha256.Sum256(archive) + if scenario == "checksum" { + sum[0]++ + } + fixtureWrite(t, filepath.Join(root, "SHA256SUMS"), fmt.Sprintf("%x archie-core-archied-v2.0.0-linux-amd64.zip\n", sum), 0o600) + fixtureWrite(t, filepath.Join(fake, "curl"), `#!/bin/bash +out="" +for ((i=1;i<=$#;i++)); do + if [ "${!i}" = -o ]; then j=$((i+1)); out="${!j}"; fi + case "${!i}" in + */SHA256SUMS) source="$FIXTURE/SHA256SUMS" ;; + *.zip) source="$FIXTURE/archive.zip" ;; + esac +done +if [ -n "$out" ]; then cp "$source" "$out"; exit; fi +if [ "$SCENARIO" = unhealthy ]; then exit 22; fi +busy=0 +[ "$SCENARIO" = busy ] && busy=1 +printf '{"status":"ok","version":"2.0.0","in_flight":%s}\n' "$busy" +`, 0o755) + for _, name := range []string{"systemctl", "systemd-run", "docker"} { + fixtureWrite(t, filepath.Join(fake, name), "#!/bin/bash\nexit 0\n", 0o755) + } + fixtureWrite(t, filepath.Join(fake, "git"), "#!/bin/bash\necho unexpected-git >&2\nexit 98\n", 0o755) + fixtureWrite(t, filepath.Join(fake, "mv"), `#!/bin/bash +/bin/mv "$@" || exit +if [ -n "$CRASH_AFTER" ] && [[ "$*" == *".archied.new"* ]]; then kill -KILL "$PPID"; fi +`, 0o755) + return updateFixture{bin: bin, state: state, installer: filepath.Join(scripts, "archie-update-install"), env: append(os.Environ(), + "PATH="+fake+":"+os.Getenv("PATH"), "FIXTURE="+root, "SCENARIO="+scenario, + "ARCHIE_BIN_DIR="+bin, "ARCHIE_CONFIG_PATH="+config, "ARCHIE_UPDATE_STATE_DIR="+state, + "ARCHIE_UPDATE_DAEMON_PREVIOUS=1.0.0", "ARCHIE_UPDATE_DAEMON_VERSION=2.0.0", + "ARCHIE_UPDATE_AGENT_PREVIOUS=1.0.0", "ARCHIE_UPDATE_AGENT_VERSION="+map[bool]string{true: "2.0.0"}[scenario == "missing-digest"], + "ARCHIE_UPDATE_HEALTH_TIMEOUT=1", "ARCHIE_UPDATE_OBSERVE_SECONDS=0", "ARCHIE_UPDATE_POLL_INTERVAL=0.01", + "ARCHIE_UPDATE_STATE_STORE_TARGET=", "CRASH_AFTER="+map[bool]string{true: "archied"}[scenario == "crash"], + )} +} + +func makeUpdateArchive(t *testing.T, root, scripts, scenario string) []byte { + t.Helper() + file, err := os.Create(filepath.Join(root, "archive.zip")) + if err != nil { + t.Fatal(err) + } + z := zip.NewWriter(file) + files := map[string]string{"release.json": `{"required_topology":{"units":["archie-state-store","archie-gateway","archie-ui","archie-messaging","archied"],"config_sections":["services.state","services.gateway"]},"agent_image":"mutable:latest"}`} + for _, name := range []string{"archied", "archie-state-store", "archie-gateway", "archie-ui", "archie-messaging"} { + files[name] = "#!/bin/bash\n# new\nif [ \"$1\" = -version ]; then echo '" + name + " 2.0.0'; fi\nexit 0\n" + } + for _, name := range []string{"archie-update-install", "archie-update-watchdog", "archie-update-check"} { + data, err := os.ReadFile(filepath.Join(scripts, name)) + if err != nil { + t.Fatal(err) + } + files[name] = string(data) + } + if scenario == "unsafe-path" { + files["../escape"] = "escape" + } + for name, body := range files { + header := &zip.FileHeader{Name: "archie-core-archied-v2.0.0-linux-amd64/" + name, Method: zip.Deflate} + header.SetMode(0o755) + w, err := z.CreateHeader(header) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(file.Name()) + if err != nil { + t.Fatal(err) + } + return data +} + +func (f updateFixture) run(t *testing.T, path string, args ...string) (string, error) { + t.Helper() + cmd := exec.CommandContext(t.Context(), path, args...) + cmd.Env = f.env + data, err := cmd.CombinedOutput() + return string(data), err +} + +func (f updateFixture) assertVersion(t *testing.T, want string) { + t.Helper() + data, err := os.ReadFile(filepath.Join(f.bin, "archied")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "# "+want) { + t.Fatalf("wanted %s executable: %s", want, data) + } +} diff --git a/internal/releaseupdate/topology.go b/internal/releaseupdate/topology.go new file mode 100644 index 000000000..2f5ed4fc6 --- /dev/null +++ b/internal/releaseupdate/topology.go @@ -0,0 +1,25 @@ +package releaseupdate + +import "fmt" + +type Topology struct { + Units []string `json:"units"` + ConfigSections []string `json:"config_sections"` +} + +// MigrationPlan preserves the release's dependency order. Unknown observations +// require every step to be checked manually rather than assuming a partial host. +func (t Topology) MigrationPlan(units, sections map[string]bool) []string { + var steps []string + for _, section := range t.ConfigSections { + if !sections[section] { + steps = append(steps, fmt.Sprintf("configure [%s]", section)) + } + } + for _, unit := range t.Units { + if !units[unit] { + steps = append(steps, fmt.Sprintf("install and enable %s.service", unit)) + } + } + return steps +} diff --git a/internal/releaseupdate/topology_test.go b/internal/releaseupdate/topology_test.go new file mode 100644 index 000000000..df68faf87 --- /dev/null +++ b/internal/releaseupdate/topology_test.go @@ -0,0 +1,25 @@ +package releaseupdate + +import ( + "reflect" + "testing" +) + +func TestMigrationPlan(t *testing.T) { + topology := Topology{Units: []string{"state", "gateway", "ui", "daemon"}, ConfigSections: []string{"services.state"}} + for _, tc := range []struct { + name string + units, sections map[string]bool + want []string + }{ + {"unknown", nil, nil, []string{"configure [services.state]", "install and enable state.service", "install and enable gateway.service", "install and enable ui.service", "install and enable daemon.service"}}, + {"missing", map[string]bool{"state": true, "daemon": true}, map[string]bool{"services.state": true}, []string{"install and enable gateway.service", "install and enable ui.service"}}, + {"complete", map[string]bool{"state": true, "gateway": true, "ui": true, "daemon": true}, map[string]bool{"services.state": true}, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := topology.MigrationPlan(tc.units, tc.sections); !reflect.DeepEqual(got, tc.want) { + t.Fatalf("got %v, want %v", got, tc.want) + } + }) + } +} diff --git a/release.json b/release.json new file mode 100644 index 000000000..c28aeea67 --- /dev/null +++ b/release.json @@ -0,0 +1,6 @@ +{ + "required_topology": { + "units": ["archie-state-store", "archie-gateway", "archie-ui", "archie-messaging", "archied"], + "config_sections": ["services.state", "services.gateway"] + } +} diff --git a/scripts/archie-update-check b/scripts/archie-update-check index 4c590de57..6a4b5bd99 100755 --- a/scripts/archie-update-check +++ b/scripts/archie-update-check @@ -46,7 +46,7 @@ readonly CURL_TIMEOUT="${ARCHIE_FORGE_TIMEOUT:-20}" readonly TOKEN="${ARCHIE_FORGE_TOKEN:-}" # base 1.7.0-4-g0736708 -> 1.7.0 -base() { printf '%s\n' "${1%%-*}"; } +base() { printf '%s\n' "$1" | sed -E 's/-[0-9]+-g[0-9a-f]+$//'; } # installed_for reads the version this binary reports for a named component. # archied stamps both its own version and the runtime version the release was @@ -72,35 +72,33 @@ curl_api() { fi } -# newest_tag_git resolves the newest tag for a component prefix from the git -# remote. No local checkout, no working-tree state, no fetch -- `ls-remote` -# queries the remote directly, so this works on a host that has never cloned -# anything. +RELEASE_CHANNEL="${ARCHIE_UPDATE_RELEASE_CHANNEL:-stable}" +PIN="${ARCHIE_UPDATE_PIN:-}" +while [ "$#" -gt 0 ]; do + case "$1" in + --channel) RELEASE_CHANNEL="$2"; shift 2 ;; + --pin) PIN="$2"; shift 2 ;; + *) echo "unknown option: $1" >&2; exit 1 ;; + esac +done +select_release() { "$BIN" select-release -channel "$RELEASE_CHANNEL" -pin "$PIN"; } newest_tag_git() { - local component="$1" tags - command -v git >/dev/null 2>&1 || return 1 - tags="$(git ls-remote --tags --refs "$SOURCE_URL" "$component/v*" 2>/dev/null)" || return 1 - [ -n "$tags" ] || return 1 - printf '%s\n' "$tags" | sed 's|.*refs/tags/||' | sed "s|^$component/v||" | sort -V | tail -1 + local tags + tags="$(git ls-remote --tags --refs "$SOURCE_URL" 'archied/v*' 'v*' 2>/dev/null)" || return 1 + printf '%s\n' "$tags" | awk '{print $2}' | select_release } - -# newest_tag_api is the API equivalent, used when git is unavailable. newest_tag_api() { - local component="$1" page payload found total - for page in 1 2 3; do - payload="$(curl_api "${API_BASE}/repos/${REPO_SLUG}/tags?per_page=100&page=${page}")" || return 1 - [ -n "$payload" ] || return 1 - found="$(printf '%s\n' "$payload" \ - | grep -o "\"name\":[[:space:]]*\"${component}/v[^\"]*\"" \ - | sed 's|.*"'"${component}"'/v||; s|"$||' | sort -V | tail -1)" - if [ -n "$found" ]; then - printf '%s\n' "$found" - return 0 - fi - total="$(printf '%s\n' "$payload" | grep -c '"name"')" - [ "${total:-0}" -lt 100 ] && break - done - return 1 + local page payload tags="" + for page in $(seq 1 100); do + payload="$(curl_api "${API_BASE}/repos/${REPO_SLUG}/tags?per_page=100&page=${page}")" || return 1 + tags+="$(jq -er '.[].name' <<< "$payload")"$'\n' + if [ "$(jq 'length' <<< "$payload")" -lt 100 ]; then + printf '%s' "$tags" | select_release + return $? + fi + done + echo "tag catalog exceeded pagination limit" >&2 + return 1 } # artifact_state reports whether a tagged release is PUBLISHED with assets, diff --git a/scripts/archie-update-install b/scripts/archie-update-install index eff3c24d7..0f738c442 100755 --- a/scripts/archie-update-install +++ b/scripts/archie-update-install @@ -16,34 +16,8 @@ BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}" CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}" export PATH="/usr/local/go/bin:$PATH" -configured_agent_image() { - local image - image="$(awk ' - /^\[containers\][[:space:]]*(#.*)?$/ { in_containers = 1; next } - /^\[/ { in_containers = 0 } - in_containers && /^[[:space:]]*image[[:space:]]*=/ { - value = $0 - sub(/^[[:space:]]*image[[:space:]]*=[[:space:]]*/, "", value) - sub(/[[:space:]]*(#.*)?$/, "", value) - if (value ~ /^".*"$/) { - sub(/^"/, "", value) - sub(/"$/, "", value) - print value - } else if (length(value) >= 2 && substr(value, 1, 1) == sprintf("%c", 39) && substr(value, length(value), 1) == sprintf("%c", 39)) { - print substr(value, 2, length(value) - 2) - } - exit - } - ' "$CONFIG_PATH" 2>/dev/null || true)" - printf '%s' "$image" -} - -AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-$(configured_agent_image)}" -AGENT_IMAGE="${AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}" -AGENT_IMAGE_BACKUP="$BIN_DIR/archie-agent-image.prev.tar" # State for the managed image, independent of archied's daemon build stamp. AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}" -AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}" DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}" # The address the daemon dials for its state, so the watchdog can wait for it to # accept connections before it starts archied. Read with a narrow awk: the @@ -67,9 +41,27 @@ configured_state_store_target() { STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-$(configured_state_store_target)}" -GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui archie-messaging" +GATEWAY_SERVICES="archie-state-store archie-gateway archie-ui archie-messaging archied" GATEWAY_BINARIES="$GATEWAY_SERVICES" +SOURCE_BUILD="${ARCHIE_UPDATE_SOURCE_BUILD:-0}" +AUTO=0 +for arg in "$@"; do + case "$arg" in + --from-source) SOURCE_BUILD=1 ;; + --auto) AUTO=1 ;; + *) echo "unknown option: $arg" >&2; exit 1 ;; + esac +done + +if [ "$AUTO" = 1 ] && [ -z "${ARCHIE_UPDATE_DAEMON_VERSION:-}${ARCHIE_UPDATE_AGENT_VERSION:-}" ]; then + snapshot="$("$(dirname "$0")/archie-update-check")" + export ARCHIE_UPDATE_DAEMON_PREVIOUS="$(jq -r '.Components[] | select(.ID == "daemon") | .Installed' <<< "$snapshot")" + export ARCHIE_UPDATE_DAEMON_VERSION="$(jq -r '.Components[] | select(.ID == "daemon") | .Available' <<< "$snapshot")" + export ARCHIE_UPDATE_AGENT_PREVIOUS="$(jq -r '.Components[] | select(.ID == "agent") | .Installed' <<< "$snapshot")" + export ARCHIE_UPDATE_AGENT_VERSION="$(jq -r '.Components[] | select(.ID == "agent") | .Available' <<< "$snapshot")" +fi + PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}" PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}" GW_TARGET="${ARCHIE_UPDATE_DAEMON_VERSION:-}" @@ -132,146 +124,138 @@ for target in "$GW_TARGET" "$RT_TARGET"; do fi done +STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}" +mkdir -p "$STATE_DIR" +chmod 700 "$STATE_DIR" +exec 9>"$STATE_DIR/lock" +if ! flock -n 9; then + echo "update locked by $(cat "$STATE_DIR/holder" 2>/dev/null); journal: $STATE_DIR/transaction" >&2 + exit 1 +fi +printf 'installer PID %s\n' "$$" > "$STATE_DIR/holder" +if [ -d "$STATE_DIR/transaction" ]; then + echo "unfinished update: run $STATE_DIR/transaction/watchdog --recover before retrying" >&2 + exit 1 +fi + tmp="$(mktemp -d)" -source_dir="$tmp/source" trap 'rm -rf "$tmp"' EXIT - -echo "==> fetching approved component releases" - -git clone --quiet --filter=blob:none --no-checkout "$SOURCE_URL" "$source_dir" -git -C "$source_dir" fetch --quiet --tags origin - -gw_commit="" -rt_commit="" -if [[ ",$components," == *,daemon,* ]]; then - gw_commit="$(git -C "$source_dir" rev-parse --verify "refs/tags/archied/v$GW^{commit}")" +source_dir="$tmp/source" +release_dir="$tmp/release" +mkdir -p "$release_dir" +release_version="${GW_TARGET:-$RT_TARGET}" +if [ -n "$GW_TARGET" ] && [ -n "$RT_TARGET" ] && [ "$GW_TARGET" != "$RT_TARGET" ]; then + echo "daemon and agent must name the same release" >&2; exit 1 fi +if [ "$SOURCE_BUILD" = 1 ]; then + tag="v$release_version" + git ls-remote --exit-code --tags "$SOURCE_URL" "refs/tags/$tag" >/dev/null || tag="archied/v$release_version" + git clone --quiet --depth=1 --branch "$tag" "$SOURCE_URL" "$source_dir" + cp "$source_dir/release.json" "$release_dir/" + GATEWAY_BINARIES="$(jq -er '.required_topology.units | join(" ")' "$release_dir/release.json")" + cp "$source_dir"/scripts/archie-update-* "$release_dir/" + if [[ ",$components," == *,daemon,* ]]; then + for cmd in $GATEWAY_BINARIES; do + [[ "$cmd" =~ ^archie[a-z0-9-]*$ ]] || { echo "invalid release unit: $cmd" >&2; exit 1; } + (cd "$source_dir" && go build -ldflags "-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$GW -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$RT" -o "$release_dir/$cmd" "./cmd/$cmd") + done + fi +else + case "$(uname -s)/$(uname -m)" in + Linux/x86_64) platform=linux-amd64 ;; + *) echo "no published artifact for $(uname -s)/$(uname -m)" >&2; exit 1 ;; + esac + name="archie-core-archied-v$release_version-$platform" + base="${ARCHIE_RELEASE_BASE_URL:-https://github.com/samcharles93/archie-core/releases/download}/v$release_version" + if ! curl -fLSs --max-time 300 "$base/$name.zip" -o "$tmp/$name.zip"; then + base="${ARCHIE_RELEASE_BASE_URL:-https://github.com/samcharles93/archie-core/releases/download}/archied/v$release_version" + curl -fLSs --max-time 300 "$base/$name.zip" -o "$tmp/$name.zip" + fi + curl -fLSs --max-time 60 "$base/SHA256SUMS" -o "$tmp/SHA256SUMS" + expected="$(awk -v name="$name.zip" '$2 == name {print $1}' "$tmp/SHA256SUMS")" + [[ "$expected" =~ ^[a-fA-F0-9]{64}$ ]] || { echo "missing or ambiguous archive checksum" >&2; exit 1; } + printf '%s %s\n' "$expected" "$tmp/$name.zip" | sha256sum --check --status || { echo "release checksum mismatch" >&2; exit 1; } + # Reject paths outside the single distribution directory before extracting. + while IFS= read -r entry; do + case "$entry" in "$name/"*) ;; *) echo "unsafe archive path: $entry" >&2; exit 1 ;; esac + case "/$entry/" in */../*|*/./*|*\\*) echo "unsafe archive path: $entry" >&2; exit 1 ;; esac + done < <(unzip -Z1 "$tmp/$name.zip") + if unzip -Z -l "$tmp/$name.zip" | awk '$1 ~ /^l/ {found=1} END {exit !found}'; then + echo "release contains symlinks" >&2; exit 1 + fi + unzip -q "$tmp/$name.zip" -d "$tmp/unpacked" + if find "$tmp/unpacked" -type l | grep -q .; then echo "release contains symlinks" >&2; exit 1; fi + cp -a "$tmp/unpacked/$name/." "$release_dir/" +fi + +# Observation and config parsing belong to the daemon, including on old hosts. +observer="$release_dir/archied" +[ -x "$observer" ] || observer="$BIN_DIR/archied" +"$observer" update-topology -config "$CONFIG_PATH" -release "$release_dir/release.json" +GATEWAY_SERVICES="$(jq -er '.required_topology.units | join(" ")' "$release_dir/release.json")" +GATEWAY_BINARIES="$GATEWAY_SERVICES" + if [[ ",$components," == *,agent,* ]]; then - rt_commit="$(git -C "$source_dir" rev-parse --verify "refs/tags/archied/v$RT^{commit}")" -fi -build_commit="$gw_commit" -[ -n "$build_commit" ] || build_commit="$rt_commit" -git -C "$source_dir" checkout --quiet --detach "$build_commit" -cd "$source_dir" - -if [[ ",$components," == *,daemon,* ]]; then - missing_units="" - for svc in $GATEWAY_SERVICES; do - [ -d "$source_dir/cmd/$svc" ] || continue - if ! systemctl --user list-unit-files "$svc.service" --no-legend 2>/dev/null | grep -q .; then - missing_units="${missing_units:+$missing_units }$svc.service" - fi - done - if [ -n "$missing_units" ]; then - echo "refusing to install archied $GW: it runs as separate processes and this host has no unit for: $missing_units" >&2 - echo "create them from deployments/systemd-user-service.md, then retry the update" >&2 - exit 1 - fi - - unmanaged_units="" - for dir in "$source_dir"/cmd/*/; do - [ -d "$dir" ] || continue - svc="$(basename "$dir")" - case " $GATEWAY_SERVICES " in - *" $svc "*) continue ;; - esac - if systemctl --user list-unit-files "$svc.service" --no-legend 2>/dev/null | grep -q .; then - unmanaged_units="${unmanaged_units:+$unmanaged_units }$svc.service" - fi - done - if [ -n "$unmanaged_units" ]; then - echo "refusing to install archied $GW: this host runs units the update does not manage: $unmanaged_units" >&2 - echo "a current install.sh writes units for: $GATEWAY_SERVICES. If the installed updater predates this release, replace $BIN_DIR/archie-update-install with scripts/archie-update-install from archied/v$GW; otherwise remove the extra unit, or add it to GATEWAY_SERVICES if this host is meant to run it." >&2 - exit 1 + if [ "$SOURCE_BUILD" = 1 ]; then + docker build --build-arg "RUNTIME_VERSION=$RT" --iidfile "$tmp/image-id" "$source_dir" + AGENT_IMAGE="$(cat "$tmp/image-id")" + else + AGENT_IMAGE="$(jq -er '.agent_image' "$release_dir/release.json")" + [[ "$AGENT_IMAGE" =~ @sha256:[a-f0-9]{64}$ ]] || { echo "refusing agent update: release has no digest-pinned image" >&2; exit 1; } + docker pull "$AGENT_IMAGE" fi fi - -if [[ ",$components," == *,daemon,* ]]; then - echo "==> snapshotting the database before the update migrates it" - mkdir -p "$(dirname "$DB_SNAPSHOT")" - if ! go run ./cmd/archie-state-store backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT"; then - echo "refusing to install $GW: could not snapshot the database named by $CONFIG_PATH, so a failed migration could not be recovered" >&2 - exit 1 - fi +if [ "$AUTO" = 1 ]; then + status="$(curl -fsS --max-time 10 "${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}/health/detailed")" + [ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || { echo "refusing auto-update: daemon has work in flight or cannot report it" >&2; exit 1; } fi -install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install" -install -m755 "$source_dir/scripts/archie-update-watchdog" "$BIN_DIR/archie-update-watchdog" -install -m755 "$source_dir/scripts/archie-update-check" "$BIN_DIR/archie-update-check" - built="" if [[ ",$components," == *,daemon,* ]]; then - ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" - ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$GW" - ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$RT" for cmd in $GATEWAY_BINARIES; do - [ -d "./cmd/$cmd" ] || continue - echo "==> building $cmd $GW" - go build -ldflags "$ldflags" -o "$tmp/$cmd" "./cmd/$cmd" - built="${built:+$built }$cmd" + reported="$("$release_dir/$cmd" -version | awk -v cmd="$cmd" '$1 == cmd {print $2; exit}')" + [ "$reported" = "$GW" ] || { echo "candidate $cmd reports $reported, expected $GW" >&2; exit 1; } done - - echo "==> backing up current binaries" - for cmd in $built; do - if [ -f "$BIN_DIR/$cmd" ]; then - cp -f "$BIN_DIR/$cmd" "$BIN_DIR/$cmd.prev" - else - rm -f "$BIN_DIR/$cmd.prev" - fi - done - echo "==> installing $built" - for cmd in $built; do - install -m755 "$tmp/$cmd" "$BIN_DIR/$cmd" - done - - if ! verify_installed_versions "$GW"; then - exit 1 - fi + mkdir -p "$(dirname "$DB_SNAPSHOT")" + "$BIN_DIR/archie-state-store" backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT" + built="$GATEWAY_BINARIES archie-update-install archie-update-watchdog archie-update-check" fi + +# Freeze recovery code and every prior file before publishing the intent. +journal="$STATE_DIR/preparing" +rm -rf "$journal" +mkdir -p "$journal/previous" +cp "$release_dir/archie-update-watchdog" "$journal/watchdog" +cp "$observer" "$journal/helper" if [[ ",$components," == *,agent,* ]]; then - git checkout --quiet --detach "$rt_commit" - echo "==> saving managed worker image $AGENT_IMAGE for rollback" - docker image save --output "$AGENT_IMAGE_BACKUP" "$AGENT_IMAGE" - if [ -f "$AGENT_VERSION_FILE" ]; then - cp -f "$AGENT_VERSION_FILE" "$AGENT_VERSION_BACKUP" - AGENT_VERSION_PREVIOUS_EXISTS=1 - else - rm -f "$AGENT_VERSION_BACKUP" - AGENT_VERSION_PREVIOUS_EXISTS=0 - fi - echo "==> rebuilding managed worker image $AGENT_IMAGE for runtime $RT" - docker build --build-arg "RUNTIME_VERSION=$RT" --tag "$AGENT_IMAGE" . - version_tmp="$(mktemp "$AGENT_VERSION_FILE.tmp.XXXXXX")" - printf '%s\n' "$RT" > "$version_tmp" - chmod 600 "$version_tmp" - mv -f "$version_tmp" "$AGENT_VERSION_FILE" + "$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -prepare "$AGENT_IMAGE" fi - -echo "==> handing off to the update watchdog for restart and health verification" +chmod 700 "$journal/watchdog" +for cmd in $built; do + if [ -f "$BIN_DIR/$cmd" ]; then cp -p "$BIN_DIR/$cmd" "$journal/previous/$cmd"; fi + install -m755 "$release_dir/$cmd" "$BIN_DIR/.$cmd.new" +done +if [ -f "$AGENT_VERSION_FILE" ]; then cp -p "$AGENT_VERSION_FILE" "$journal/agent.version"; fi +# No unit/config changes are made: topology migration is explicitly refused. +export ARCHIE_UPDATE_AUTO="$AUTO" +export ARCHIE_UPDATE_STATE_DIR="$STATE_DIR" +export ARCHIE_BIN_DIR="$BIN_DIR" ARCHIE_CONFIG_PATH="$CONFIG_PATH" +export ARCHIE_UPDATE_COMPONENTS="$components" ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" ARCHIE_UPDATE_BINARIES="$built" +export ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT" +export ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT" +export ARCHIE_AGENT_VERSION_FILE="$AGENT_VERSION_FILE" ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT" +export ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET" +# Bash's quoted declarations preserve spaces without interpreting journal values. +while IFS= read -r var; do declare -p "$var"; done < <(compgen -e | grep -E '^ARCHIE_(UPDATE_|BIN_DIR$|CONFIG_PATH$|HEALTH_URL$|AGENT_VERSION_FILE$|DB_SNAPSHOT$)') > "$journal/environment" +printf 'prepared\n' > "$journal/phase" +sync -f "$BIN_DIR" +sync -f "$journal" +mv "$journal" "$STATE_DIR/transaction" +sync -f "$STATE_DIR" +journal="$STATE_DIR/transaction" +# The independent watchdog holds the lock and owns promotion, restart and rollback. systemd-run --user --unit=archied-update-restart --collect --quiet \ - --setenv=ARCHIE_BIN_DIR="$BIN_DIR" \ - --setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-}" \ - --setenv=ARCHIE_UPDATE_REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" \ - --setenv=ARCHIE_UPDATE_CHANNEL="${ARCHIE_UPDATE_CHANNEL:-}" \ - --setenv=ARCHIE_UPDATE_CHAT_ID="${ARCHIE_UPDATE_CHAT_ID:-0}" \ - --setenv=ARCHIE_UPDATE_THREAD_ID="${ARCHIE_UPDATE_THREAD_ID:-0}" \ - --setenv=ARCHIE_UPDATE_COMPONENTS="$components" \ - --setenv=ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" \ - --setenv=ARCHIE_UPDATE_BINARIES="$built" \ - --setenv=ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" \ - --setenv=ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT" \ - --setenv=ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" \ - --setenv=ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT" \ - --setenv=ARCHIE_AGENT_IMAGE="$AGENT_IMAGE" \ - --setenv=ARCHIE_AGENT_IMAGE_BACKUP="$AGENT_IMAGE_BACKUP" \ - --setenv=ARCHIE_AGENT_VERSION_FILE="$AGENT_VERSION_FILE" \ - --setenv=ARCHIE_AGENT_VERSION_BACKUP="$AGENT_VERSION_BACKUP" \ - --setenv=ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS="${AGENT_VERSION_PREVIOUS_EXISTS:-0}" \ - --setenv=ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT" \ - --setenv=ARCHIE_CONFIG_PATH="$CONFIG_PATH" \ - --setenv=ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET" \ - "$BIN_DIR/archie-update-watchdog" - + --setenv=ARCHIE_UPDATE_STATE_DIR="$STATE_DIR" "$journal/watchdog" case "$components" in daemon) previous="{\"daemon\":\"$(json_escape "$PREV_GW")\"}" diff --git a/scripts/archie-update-watchdog b/scripts/archie-update-watchdog index d3b088399..dc4299197 100755 --- a/scripts/archie-update-watchdog +++ b/scripts/archie-update-watchdog @@ -7,36 +7,39 @@ # It does not rebuild or reinstall the gateway binary. set -uo pipefail -BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}" -AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}" -AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}" -AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}" -AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}" -AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}" -DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}" -CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}" -# The daemon exports its own health address (releaseupdate.CommandInstaller); -# this default only covers a hand-driven run, and must stay the address archied -# itself serves /healthz on -- [health] listen, defaulted in -# internal/infrastructure/configuration/defaults.go. +STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}" +journal="$STATE_DIR/transaction" +mkdir -p "$STATE_DIR" +exec 9>"$STATE_DIR/lock" +if [ "${1:-}" = --recover ]; then + flock -n 9 || exit 0 # A live installer/watchdog owns recovery. +else + flock -w 60 9 || { echo "update locked by $(cat "$STATE_DIR/holder")" >&2; exit 1; } +fi +[ -d "$journal" ] || exit 0 +printf 'watchdog PID %s\n' "$$" > "$STATE_DIR/holder" +source "$journal/environment" +BIN_DIR="$ARCHIE_BIN_DIR" +CONFIG_PATH="$ARCHIE_CONFIG_PATH" +DB_SNAPSHOT="$ARCHIE_DB_SNAPSHOT" +AGENT_VERSION_FILE="$ARCHIE_AGENT_VERSION_FILE" HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}" HEALTH_URL="${HEALTH_URL%/}" REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}" -POLL_INTERVAL_SECONDS=3 -COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}" -# Set by archie-update-install: the units to restart, in start order, and the -# binaries it replaced. The defaults keep a hand-driven run on a pre- -# decomposition host working. -UNITS="${ARCHIE_UPDATE_UNITS:-archied}" -BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}" - -# The State Store's address, so the start order below can mean "ready" rather than -# "launched". Empty when the config names none or the installed updater predates -# passing one -- waiting is then skipped, which is the previous behaviour. +OBSERVE_SECONDS="${ARCHIE_UPDATE_OBSERVE_SECONDS:-120}" +POLL_INTERVAL_SECONDS="${ARCHIE_UPDATE_POLL_INTERVAL:-3}" +COMPONENTS="$ARCHIE_UPDATE_COMPONENTS" +UNITS="$ARCHIE_UPDATE_UNITS" +BINARIES="$ARCHIE_UPDATE_BINARIES" STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-}" STATE_STORE_TIMEOUT_SECONDS="${ARCHIE_UPDATE_STATE_STORE_TIMEOUT:-30}" +phase() { + printf '%s\n' "$1" > "$journal/phase.new" && sync -f "$journal" && + mv "$journal/phase.new" "$journal/phase" && sync -f "$journal" +} + json_escape() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' } @@ -75,10 +78,22 @@ EOF mv -f "$tmp" "$REPORT_PATH" } +healthy() { + local report + report="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 1 + [ "$(jq -r '.version' <<< "$report")" = "$ARCHIE_UPDATE_INSTALLED_GATEWAY" ] && + [ "$(jq -r '.status' <<< "$report")" = ok ] +} + wait_healthy() { local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS)) while [ "$SECONDS" -lt "$deadline" ]; do - if curl -sf -o /dev/null "$HEALTH_URL/healthz"; then + if healthy; then + deadline=$((SECONDS + OBSERVE_SECONDS)) + while [ "$SECONDS" -lt "$deadline" ]; do + sleep "$POLL_INTERVAL_SECONDS" + healthy || return 1 + done return 0 fi sleep "$POLL_INTERVAL_SECONDS" @@ -86,20 +101,7 @@ wait_healthy() { return 1 } -# manage_unit runs one systemctl action. archied is required: this whole -# mechanism exists to restart it, and a failed stop must abort the rollback -# rather than let it race a live process over the binary and the database. -# The sibling processes are best-effort, because archie-update-install has -# already refused the update if a unit for one of them was missing, and a -# probe that guesses wrong must not silently skip the required stop. -manage_unit() { - local action="$1" unit="$2" - if [ "$unit" = archied ]; then - systemctl --user "$action" "$unit.service" - return - fi - systemctl --user "$action" "$unit.service" || true -} +manage_unit() { systemctl --user "$1" "$2.service"; } # port_accepts reports whether host:port accepts a connection. bash's /dev/tcp # costs no process and no dependency, and accepting is exactly the signal this @@ -109,21 +111,7 @@ port_accepts() { (exec 3<>"/dev/tcp/${1%:*}/${1##*:}") 2>/dev/null } -# wait_for_state_store blocks until the State Store accepts connections, or the -# budget runs out. -# -# Ordering the units is not enough on its own: `systemctl --user start` returns -# once the process is forked, not once its port accepts. Measured on a 1.37.1 -> -# 1.38.0 update, the State Store's unit reported started while it was still -# binding, archied was started next, dialled a closed port and exited 1, and only -# systemd's Restart=on-failure brought it back seven seconds later. A host whose -# unit lacks that restart policy gets a failed update behind a healthy-looking -# "build and install succeeded" line. -# -# A timeout does NOT fail the update. If the State Store never comes up, archied -# fails closed on its own and wait_healthy rolls the release back -- waiting is an -# optimisation for the ordinary case, and losing the race must stay a slower path -# to the same place rather than a second failure mode. +# Wait for the dependency before starting its callers. wait_for_state_store() { [ -n "$STATE_STORE_TARGET" ] || { echo "==> no State Store target known; starting the rest without waiting for it" >&2 @@ -169,39 +157,70 @@ stop_units() { return $rc } -# rollback_gateway restores every binary the install replaced. A binary with -# no .prev was introduced by the failed release, so it is removed rather than -# left behind pointing at processes the restored release does not run. -rollback_gateway() { - local cmd rc=0 +rollback_files() { + local cmd for cmd in $BINARIES; do - if [ -f "$BIN_DIR/$cmd.prev" ]; then - cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1 + if [ -f "$journal/previous/$cmd" ]; then + cp -p "$journal/previous/$cmd" "$BIN_DIR/.$cmd.rollback" && + mv -f "$BIN_DIR/.$cmd.rollback" "$BIN_DIR/$cmd" || return 1 else - rm -f "$BIN_DIR/$cmd" || rc=1 + rm -f "$BIN_DIR/$cmd" || return 1 fi + rm -f "$BIN_DIR/.$cmd.new" done - return $rc + if [[ ",$COMPONENTS," == *,agent,* ]]; then + if [ -f "$journal/agent.version" ]; then + cp -p "$journal/agent.version" "$AGENT_VERSION_FILE.new" && + mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1 + else + rm -f "$AGENT_VERSION_FILE" || return 1 + fi + fi + sync -f "$BIN_DIR" } -rollback_agent_image() { - if [ ! -f "$AGENT_IMAGE_BACKUP" ]; then - echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2 - return 1 - fi - docker image load --input "$AGENT_IMAGE_BACKUP" +finish() { + phase "$1" || return 1 + # A terminal journal is harmless if power fails before removal. + rm -rf "$STATE_DIR/finished" + mv "$journal" "$STATE_DIR/finished" && sync -f "$STATE_DIR" || return 1 + rm -rf "$STATE_DIR/finished" } -# The sidecar describes the image restored above. Restore its prior state in -# the same rollback path; a first install has no prior file and must remove the -# newly-created metadata rather than leave a false installed version behind. -rollback_agent_version() { - if [ "$AGENT_VERSION_PREVIOUS_EXISTS" = 1 ]; then - [ -f "$AGENT_VERSION_BACKUP" ] || return 1 - cp -f "$AGENT_VERSION_BACKUP" "$AGENT_VERSION_FILE" - else - rm -f "$AGENT_VERSION_FILE" +rollback() { + phase rolling-back || return 1 + stop_units && rollback_files || return 1 + if [ -f "$journal/image.json" ]; then + manage_unit start archie-state-store && wait_for_state_store && + "$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -rollback || return 1 fi + phase rolled-back || return 1 + start_units || return 1 + write_report failed true || return 1 + finish rolled-back +} + +promote() { + if [ "${ARCHIE_UPDATE_AUTO:-0}" = 1 ]; then + local status + status="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 2 + [ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || return 2 + fi + phase switching || return 1 + stop_units || return 1 + for cmd in $BINARIES; do + mv -f "$BIN_DIR/.$cmd.new" "$BIN_DIR/$cmd" || return 1 + done + if [[ ",$COMPONENTS," == *,agent,* ]]; then + printf '%s\n' "$ARCHIE_UPDATE_INSTALLED_RUNTIME" > "$AGENT_VERSION_FILE.new" && + mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1 + fi + sync -f "$BIN_DIR" || return 1 + if [ -f "$journal/image.json" ]; then + manage_unit start archie-state-store && wait_for_state_store && + "$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" || return 1 + fi + phase observing && start_units && wait_healthy } # explain_database_recovery states what a daemon rollback does not do. The @@ -214,62 +233,25 @@ explain_database_recovery() { echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2 } -start_units - -if wait_healthy; then - echo "==> new version is healthy" - write_report "passed" false +case "$(cat "$journal/phase")" in + committed) start_units && finish committed; exit $? ;; + rolled-back) start_units && finish rolled-back; exit $? ;; +esac +if [ -z "${1:-}" ] && promote; then + write_report passed false && finish committed + exit $? +fi +if [ "$(cat "$journal/phase")" = prepared ]; then + echo "update was not promoted; leaving running work alone" >&2 + for cmd in $BINARIES; do rm -f "$BIN_DIR/.$cmd.new"; done + write_report failed false && finish rolled-back || exit 1 + if [ "${1:-}" = --boot-recover ]; then start_units; exit $?; fi + exit 1 +fi +echo "==> restoring interrupted or unhealthy update" >&2 +if rollback; then + if [[ ",$COMPONENTS," == *,daemon,* ]]; then explain_database_recovery; fi exit 0 fi - -echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components" -rolled_back=false -# Stop before touching the executable. The service uses Restart=on-failure in -# the supported unit, so copying rollback state while a crash-loop child is -# alive can execute the new binary after its files have been replaced. -if stop_units; then - case "$COMPONENTS" in - daemon) - if [ -f "$BIN_DIR/archied.prev" ]; then - if rollback_gateway; then - rolled_back=true - fi - else - echo "==> no backup for archied; cannot roll it back automatically" >&2 - fi - ;; - agent) - if [ -f "$AGENT_IMAGE_BACKUP" ]; then - if rollback_agent_image && rollback_agent_version; then - rolled_back=true - fi - else - echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2 - fi - ;; - daemon,agent) - # Preserve the old all-or-nothing rollback guarantee: never restore one - # component and leave the other on the failed release. - if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ]; then - if rollback_agent_image && rollback_agent_version && rollback_gateway; then - rolled_back=true - fi - else - echo "==> not all changed-component backups exist; cannot roll back automatically" >&2 - fi - ;; - *) - echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2 - ;; - esac -else - echo "==> could not stop every unit; refusing rollback while one may be running" >&2 -fi -if [ "$rolled_back" = true ]; then - start_units -fi -if [[ ",$COMPONENTS," == *,daemon,* ]]; then - explain_database_recovery -fi - -write_report "failed" "$rolled_back" +echo "rollback incomplete; retained journal: $journal" >&2 +exit 1