feat(releaseupdate): install from release artifacts with journaled rollback, topology check and channels
This commit is contained in:
24 files changed
+1186
-333
No files matched your search
@@ -166,6 +166,8 @@ jobs:
|
||||
fi
|
||||
|
||||
build-and-push:
|
||||
outputs:
|
||||
agent_digest: ${{ steps.agent.outputs.digest }}
|
||||
needs: gate
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -258,6 +260,7 @@ jobs:
|
||||
cache-to: type=gha,mode=max,scope=archied
|
||||
|
||||
- name: Build and push archie-agent
|
||||
id: agent
|
||||
if: steps.versions.outputs.runtime_tag != '' || steps.versions.outputs.release_tag != ''
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
@@ -282,7 +285,7 @@ jobs:
|
||||
# the Docker image build-and-push already publishes, never as a host
|
||||
# binary (archie-core-8cda.4.7.1).
|
||||
dist-zip:
|
||||
needs: gate
|
||||
needs: [gate, build-and-push]
|
||||
# Not gated on github.ref. A release is pushed as main plus both tags at
|
||||
# one SHA, the concurrency group below collapses those into a single run,
|
||||
# and the survivor is whichever ref happened to start last -- for
|
||||
@@ -329,7 +332,7 @@ jobs:
|
||||
# list and the copy list below stop matching the commands in cmd/.
|
||||
ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=${{ steps.version.outputs.version }} -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=${{ steps.version.outputs.version }}"
|
||||
mkdir -p dist
|
||||
for cmd in archied archie-gateway archie-state-store archie-ui archie-messaging; do
|
||||
for cmd in $(jq -r '.required_topology.units[]' release.json); do
|
||||
go build -ldflags "$ldflags" -o "dist/$cmd" "./cmd/$cmd"
|
||||
done
|
||||
|
||||
@@ -339,19 +342,28 @@ jobs:
|
||||
set -e
|
||||
name="archie-core-archied-v${{ steps.version.outputs.version }}-linux-amd64"
|
||||
mkdir -p "$name"
|
||||
cp dist/archied dist/archie-gateway dist/archie-state-store dist/archie-ui dist/archie-messaging "$name/"
|
||||
for cmd in $(jq -r '.required_topology.units[]' release.json); do
|
||||
cp "dist/$cmd" "$name/"
|
||||
done
|
||||
cp deployments/INSTRUCTIONS.md CHANGELOG.md "$name/"
|
||||
cp scripts/archie-update-* "$name/"
|
||||
digest='${{ needs.build-and-push.outputs.agent_digest }}'
|
||||
test -n "$digest"
|
||||
jq --arg image "ghcr.io/samcharles93/archie-agent@$digest" '. + {agent_image: $image}' release.json > "$name/release.json"
|
||||
# README.md doesn't exist yet in this repo; include it once it does
|
||||
# rather than fail the release on its absence.
|
||||
[ -f README.md ] && cp README.md "$name/"
|
||||
zip -r "$name.zip" "$name"
|
||||
sha256sum "$name.zip" > SHA256SUMS
|
||||
echo "name=$name" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload distribution zip
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ steps.zip.outputs.name }}
|
||||
path: ${{ steps.zip.outputs.name }}.zip
|
||||
path: |
|
||||
${{ steps.zip.outputs.name }}.zip
|
||||
SHA256SUMS
|
||||
retention-days: 90
|
||||
|
||||
# A Release is what an operator can actually download from; the
|
||||
@@ -370,7 +382,10 @@ jobs:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -e
|
||||
tag="archied/v${{ steps.version.outputs.version }}"
|
||||
tag="v${{ steps.version.outputs.version }}"
|
||||
if ! git ls-remote --exit-code --tags origin "refs/tags/$tag" >/dev/null; then
|
||||
tag="archied/v${{ steps.version.outputs.version }}"
|
||||
fi
|
||||
version="${{ steps.version.outputs.version }}"
|
||||
# The release body is this version's changelog section, not the whole
|
||||
# file: from its "## [x.y.z]" heading to the next "## [" heading.
|
||||
@@ -391,11 +406,11 @@ jobs:
|
||||
# alone, so a re-run of this job (a retried or superseded deploy)
|
||||
# cannot fail on an existing Release or duplicate the zip.
|
||||
if gh release view "$tag" >/dev/null 2>&1; then
|
||||
gh release upload "$tag" "${{ steps.zip.outputs.name }}.zip" --clobber
|
||||
gh release upload "$tag" "${{ steps.zip.outputs.name }}.zip" SHA256SUMS --clobber
|
||||
else
|
||||
gh release create "$tag" \
|
||||
--title "archied v$version" \
|
||||
--notes-file "$notes" \
|
||||
"${{ steps.zip.outputs.name }}.zip"
|
||||
"${{ steps.zip.outputs.name }}.zip" SHA256SUMS
|
||||
fi
|
||||
echo "published $tag with ${{ steps.zip.outputs.name }}.zip"
|
||||
@@ -16,6 +16,15 @@ func main() {
|
||||
if archied.IsSetupArgs(args) {
|
||||
os.Exit(archied.RunSetup(args[1:], os.Stdin, os.Stdout, os.Stderr))
|
||||
}
|
||||
if len(args) > 0 && args[0] == "select-release" {
|
||||
os.Exit(archied.RunReleaseChannel(args[1:], os.Stdin, os.Stdout, os.Stderr))
|
||||
}
|
||||
if len(args) > 0 && args[0] == "update-image" {
|
||||
os.Exit(archied.RunUpdateImage(args[1:], os.Stderr))
|
||||
}
|
||||
if len(args) > 0 && args[0] == "update-topology" {
|
||||
os.Exit(archied.RunUpdateTopology(args[1:], os.Stdout, os.Stderr))
|
||||
}
|
||||
if len(args) > 0 && args[0] == "status" {
|
||||
os.Exit(archied.RunStatus(args[1:], os.Stdout, os.Stderr))
|
||||
}
|
||||
|
||||
@@ -86,3 +86,45 @@ only `archied` and `archie-agent` are independently versioned).
|
||||
carries the Telegram, email and webhook channels since the v1.30.0 extraction, so
|
||||
an install (or an upgrade across v1.30.0) needs its unit as well -- see
|
||||
`deployments/systemd-user-service.md`.
|
||||
|
||||
### Native self-update
|
||||
|
||||
`archie-update-install` downloads the approved release's Linux/amd64 archive
|
||||
and verifies its `SHA256SUMS` entry before executing or installing it. Releases
|
||||
must include `release.json`, all host binaries, and the three updater scripts.
|
||||
Older releases without this contract cannot be installed through the artifact
|
||||
path. `--from-source` (or `ARCHIE_UPDATE_SOURCE_BUILD=1`) explicitly clones and
|
||||
builds the selected release instead.
|
||||
|
||||
The update check command accepts `--channel stable` (default), `--channel next`
|
||||
(includes prereleases), or `--channel exact-pin --pin 1.2.3`. Set these arguments
|
||||
in the control plane's channel settings alongside the existing check command.
|
||||
The notification destination remains separate from the release channel.
|
||||
|
||||
For scheduled, unattended runs, `archie-update-install --auto` discovers the
|
||||
selected release and refuses if the daemon reports work in flight. It checks
|
||||
again immediately before promotion. A scheduler can set
|
||||
`ARCHIE_UPDATE_RELEASE_CHANNEL` and `ARCHIE_UPDATE_PIN`; nothing schedules updates
|
||||
by default. All services stop through their ordinary graceful shutdown path.
|
||||
|
||||
`release.json` supplies the service order and required config sections to both
|
||||
first-install and self-update. A missing unit or config section produces an
|
||||
ordered migration plan and refuses installation. Apply that plan before retrying.
|
||||
Automatic topology edits are intentionally unsupported.
|
||||
|
||||
The watchdog switches staged executables by rename, restarts services, and
|
||||
requires the candidate daemon's existing readiness registry to remain healthy
|
||||
for 120 seconds. Agent images come from the release manifest by digest. The
|
||||
image switch updates the control-plane policy, preserving other settings;
|
||||
rollback restores the prior image policy with conflict detection. Source builds
|
||||
use the built image's immutable image ID.
|
||||
|
||||
Transactions live in `${XDG_STATE_HOME:-~/.local/state}/archie/update`, outside
|
||||
the binary directory. `phase`, `environment`, `image.json` (when applicable),
|
||||
and `previous/` describe the transaction. A lock reports the owning process.
|
||||
On daemon boot, an interrupted transaction schedules independent recovery before
|
||||
normal startup. To recover manually, run `transaction/watchdog --recover` from
|
||||
that state directory. A failed recovery keeps the journal for another attempt.
|
||||
|
||||
Database snapshots remain available for manual recovery. Schema migrations and
|
||||
PostgreSQL data are never automatically reversed.
|
||||
@@ -84,7 +84,6 @@ require (
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 // indirect
|
||||
@@ -119,8 +118,10 @@ require (
|
||||
go.lsp.dev/jsonrpc2 v1.0.1
|
||||
go.lsp.dev/protocol v1.0.1
|
||||
go.lsp.dev/uri v1.0.1
|
||||
golang.org/x/mod v0.41.0
|
||||
golang.org/x/net v0.59.0
|
||||
golang.org/x/oauth2 v0.37.0
|
||||
golang.org/x/sys v0.48.0
|
||||
golang.org/x/term v0.46.0
|
||||
google.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e
|
||||
google.golang.org/protobuf v1.36.12
|
||||
|
||||
@@ -270,6 +270,8 @@ golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU=
|
||||
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
|
||||
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
||||
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
|
||||
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
|
||||
|
||||
+8
-5
@@ -112,7 +112,7 @@ if [ "$(uname -s)" != "Linux" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for cmd in git go; do
|
||||
for cmd in git go jq; do
|
||||
if ! command -v "$cmd" &>/dev/null; then
|
||||
echo "Error: '$cmd' is required but not installed." >&2
|
||||
exit 1
|
||||
@@ -159,6 +159,8 @@ else
|
||||
fi
|
||||
fi
|
||||
|
||||
TOPOLOGY_UNITS="$(jq -er '.required_topology.units | join(" ")' "$SRC_DIR/release.json")"
|
||||
|
||||
# 4. Build and install the native daemon. archie-agent is deployed only as
|
||||
# the managed task image; installing a host binary would imply an unsupported
|
||||
# host execution path.
|
||||
@@ -193,10 +195,10 @@ echo "==> Building native archie binaries..."
|
||||
# webhook channels dead with no error anywhere. This list,
|
||||
# the zip's two lists and the two in scripts/archie-update-install must agree;
|
||||
# TestDistZipShipsEveryHostCommand fails when they do not.
|
||||
for cmd in archied archie-gateway archie-state-store archie-ui archie-messaging; do
|
||||
for cmd in $TOPOLOGY_UNITS; do
|
||||
go build -ldflags "${LDFLAGS}" -o "${ARCHIE_BIN_DIR}/${cmd}" "./cmd/${cmd}"
|
||||
done
|
||||
install -m755 "${SRC_DIR}/scripts/archie-update-install" "${ARCHIE_BIN_DIR}/archie-update-install"
|
||||
install -m755 "${SRC_DIR}"/scripts/archie-update-* "${ARCHIE_BIN_DIR}/"
|
||||
)
|
||||
echo " Installed archied, archie-gateway, archie-state-store, archie-ui, archie-messaging and updater to ${ARCHIE_BIN_DIR}/"
|
||||
|
||||
@@ -520,7 +522,8 @@ EOF
|
||||
|
||||
systemctl --user daemon-reload || true
|
||||
SERVICE_INSTALLED=true
|
||||
for unit in archied archie-state-store archie-gateway archie-ui archie-messaging; do
|
||||
for unit in $TOPOLOGY_UNITS; do
|
||||
test -f "${SYSTEMD_USER_DIR}/${unit}.service" || { echo "missing required unit: $unit" >&2; exit 1; }
|
||||
echo " Installed ${SYSTEMD_USER_DIR}/${unit}.service"
|
||||
done
|
||||
|
||||
@@ -546,7 +549,7 @@ EOF
|
||||
# not race a dial it depends on.
|
||||
if [ "${AUTO_START}" = true ]; then
|
||||
echo "==> Enabling and starting Archie services..."
|
||||
systemctl --user enable --now archie-state-store archie-gateway archied archie-ui archie-messaging ||
|
||||
systemctl --user enable --now $TOPOLOGY_UNITS ||
|
||||
echo " Notice: Could not start every service automatically. Check: systemctl --user status archied archie-state-store archie-gateway archie-ui archie-messaging"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||
"github.com/samcharles93/archie-core/internal/buildinfo"
|
||||
"github.com/samcharles93/archie-core/internal/domain/health"
|
||||
)
|
||||
|
||||
@@ -24,6 +25,7 @@ import (
|
||||
type healthSurface struct {
|
||||
serving atomic.Bool
|
||||
registry func() *health.Registry
|
||||
inFlight func() int
|
||||
}
|
||||
|
||||
// markServing tolerates a nil surface so a boot path that serves no health
|
||||
@@ -70,7 +72,15 @@ func (h *healthSurface) handleDetailed(w http.ResponseWriter, r *http.Request) {
|
||||
if report.Status != health.StatusOK {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(report)
|
||||
inFlight := -1
|
||||
if h.serving.Load() && h.inFlight != nil {
|
||||
inFlight = h.inFlight()
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(struct {
|
||||
health.Report
|
||||
Version string `json:"version"`
|
||||
InFlight int `json:"in_flight"`
|
||||
}{Report: report, Version: buildinfo.Version, InFlight: inFlight})
|
||||
}
|
||||
|
||||
// startHealth binds the daemon's liveness surface on [health].listen. A
|
||||
@@ -78,7 +88,15 @@ func (h *healthSurface) handleDetailed(w http.ResponseWriter, r *http.Request) {
|
||||
// updates verified, and the watchdog would roll back every release it
|
||||
// installs.
|
||||
func (b *boot) startHealth(ctx context.Context) error {
|
||||
b.health = &healthSurface{registry: func() *health.Registry { return b.healthRegistry }}
|
||||
b.health = &healthSurface{
|
||||
registry: func() *health.Registry { return b.healthRegistry },
|
||||
inFlight: func() int {
|
||||
if b.d == nil {
|
||||
return -1
|
||||
}
|
||||
return b.d.InFlight()
|
||||
},
|
||||
}
|
||||
addr := b.cfg.Health.Listen
|
||||
if err := b.serveHealth(ctx, addr, b.health.handler(), "daemon health"); err != nil {
|
||||
b.log.Error("health listener failed", "addr", addr, "err", err)
|
||||
|
||||
@@ -123,6 +123,13 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
|
||||
return 0
|
||||
}
|
||||
|
||||
if recovering, err := resumeUpdate(context.Background()); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
} else if recovering {
|
||||
return 0
|
||||
}
|
||||
|
||||
root, cancelRoot := context.WithCancel(context.Background())
|
||||
ctx, stop := signal.NotifyContext(root, os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
||||
)
|
||||
|
||||
func RunReleaseChannel(args []string, in io.Reader, out, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("select-release", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
channel := flags.String("channel", "stable", "stable, next, or exact-pin")
|
||||
pin := flags.String("pin", "", "exact release version")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 1
|
||||
}
|
||||
data, err := io.ReadAll(in)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
version, err := releaseupdate.SelectRelease(strings.Fields(string(data)), *channel, *pin)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintln(out, version)
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
||||
)
|
||||
|
||||
// RunUpdateTopology observes the host using the candidate's topology contract.
|
||||
func RunUpdateTopology(args []string, out, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("update-topology", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
configPath := flags.String("config", "", "bootstrap config file")
|
||||
releasePath := flags.String("release", "", "release manifest")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 1
|
||||
}
|
||||
if err := checkUpdateTopology(*configPath, *releasePath, out); err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func checkUpdateTopology(configPath, releasePath string, out io.Writer) error {
|
||||
data, err := os.ReadFile(releasePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var manifest struct {
|
||||
Required releaseupdate.Topology `json:"required_topology"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(manifest.Required.Units) == 0 {
|
||||
return errors.New("release has no required_topology")
|
||||
}
|
||||
for _, unit := range manifest.Required.Units {
|
||||
if !regexp.MustCompile(`^archie[a-z0-9-]*$`).MatchString(unit) {
|
||||
return fmt.Errorf("invalid release unit %q", unit)
|
||||
}
|
||||
}
|
||||
data, err = os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var document map[string]any
|
||||
if err := toml.Unmarshal(data, &document); err != nil {
|
||||
return fmt.Errorf("observe topology: %w", err)
|
||||
}
|
||||
sections := make(map[string]bool)
|
||||
for _, section := range manifest.Required.ConfigSections {
|
||||
sections[section] = hasConfigSection(document, section)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
output, observeErr := exec.CommandContext(ctx, "systemctl", "--user", "list-unit-files", "archie*.service", "--no-legend", "--no-pager").Output()
|
||||
units := observedUnits(output, observeErr)
|
||||
plan := manifest.Required.MigrationPlan(units, sections)
|
||||
if len(plan) != 0 {
|
||||
fmt.Fprintln(out, "Required migration, in order:")
|
||||
for _, step := range plan {
|
||||
fmt.Fprintln(out, " -", step)
|
||||
}
|
||||
if observeErr != nil {
|
||||
return fmt.Errorf("topology unknown; apply the printed migration before retrying: %w", observeErr)
|
||||
}
|
||||
return errors.New("refusing partial installation; apply the printed migration before retrying")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasConfigSection(document map[string]any, section string) bool {
|
||||
for part := range strings.SplitSeq(section, ".") {
|
||||
next, ok := document[part].(map[string]any)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
document = next
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func updateStateDir() string {
|
||||
if dir := os.Getenv("ARCHIE_UPDATE_STATE_DIR"); dir != "" {
|
||||
return dir
|
||||
}
|
||||
dir := os.Getenv("XDG_STATE_HOME")
|
||||
if dir == "" {
|
||||
home, _ := os.UserHomeDir()
|
||||
dir = filepath.Join(home, ".local", "state")
|
||||
}
|
||||
return filepath.Join(dir, "archie", "update")
|
||||
}
|
||||
|
||||
// Recovery runs independently of the daemon unit: it must be able to stop that
|
||||
// unit and replace its executable without systemd killing the recovery process.
|
||||
func resumeUpdate(ctx context.Context) (bool, error) {
|
||||
journal := filepath.Join(updateStateDir(), "transaction")
|
||||
if _, err := os.Stat(journal); errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
lock, err := os.OpenFile(filepath.Join(updateStateDir(), "lock"), os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer lock.Close()
|
||||
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB); errors.Is(err, unix.EWOULDBLOCK) {
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = unix.Flock(int(lock.Fd()), unix.LOCK_UN) }()
|
||||
cmd := exec.CommandContext(ctx, "systemd-run", "--user", "--collect", "--quiet", "--unit=archied-update-recovery", "--setenv=ARCHIE_UPDATE_STATE_DIR="+updateStateDir(), filepath.Join(journal, "watchdog"), "--boot-recover")
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return false, fmt.Errorf("schedule update recovery: %w: %s", err, output)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func observedUnits(output []byte, err error) map[string]bool {
|
||||
units := make(map[string]bool)
|
||||
if err != nil {
|
||||
return units
|
||||
}
|
||||
for line := range strings.SplitSeq(string(output), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 2 && fields[1] != "masked" {
|
||||
units[strings.TrimSuffix(fields[0], ".service")] = true
|
||||
}
|
||||
}
|
||||
return units
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
)
|
||||
|
||||
type imageChange struct {
|
||||
Previous string `json:"previous"`
|
||||
Candidate string `json:"candidate"`
|
||||
}
|
||||
|
||||
// RunUpdateImage journals and switches the control-plane image policy. It only
|
||||
// changes the image field, preserving concurrent edits to unrelated settings.
|
||||
func RunUpdateImage(args []string, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("update-image", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
configPath := flags.String("config", "", "bootstrap config")
|
||||
journal := flags.String("journal", "", "transaction directory")
|
||||
candidate := flags.String("prepare", "", "candidate image digest")
|
||||
rollback := flags.Bool("rollback", false, "restore the prior image")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 1
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := updateImage(ctx, *configPath, *journal, *candidate, *rollback); err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func updateImage(ctx context.Context, configPath, journal, candidate string, rollback bool) error {
|
||||
_, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), configPath, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, doc.Config.Services, secret.NewRegistry())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
rpc := client.ControlPlane()
|
||||
response, err := rpc.Query(ctx, &pb.QueryRequest{Kind: controlplane.ContainerRuntimePoliciesKind})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var policy map[string]json.RawMessage
|
||||
if err := json.Unmarshal(response.Resource.ValueJson, &policy); err != nil {
|
||||
return err
|
||||
}
|
||||
var current string
|
||||
if err := json.Unmarshal(policy["image"], ¤t); err != nil {
|
||||
return err
|
||||
}
|
||||
path := filepath.Join(journal, "image.json")
|
||||
if candidate != "" {
|
||||
return prepareImageChange(path, current, candidate)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var change imageChange
|
||||
if err := json.Unmarshal(data, &change); err != nil {
|
||||
return err
|
||||
}
|
||||
expected, desired := change.Previous, change.Candidate
|
||||
if rollback {
|
||||
expected, desired = desired, expected
|
||||
}
|
||||
if current == desired {
|
||||
return nil
|
||||
}
|
||||
if current != expected {
|
||||
return errors.New("image policy changed during update; refusing to overwrite it")
|
||||
}
|
||||
policy["image"], err = json.Marshal(desired)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err = json.Marshal(policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = rpc.Command(ctx, &pb.CommandRequest{Kind: controlplane.ContainerRuntimePoliciesKind, Command: "replace", ValueJson: data, ExpectedVersion: response.Resource.Version, Actor: "self-update", Source: "installer", RequestId: fmt.Sprintf("self-update-%x-%d", sha256.Sum256(data), response.Resource.Version)})
|
||||
return err
|
||||
}
|
||||
|
||||
func prepareImageChange(path, current, candidate string) error {
|
||||
if !regexp.MustCompile(`^([a-zA-Z0-9./:_-]+@)?sha256:[a-f0-9]{64}$`).MatchString(candidate) {
|
||||
return errors.New("candidate image must be pinned by digest")
|
||||
}
|
||||
data, err := json.Marshal(imageChange{Previous: current, Candidate: candidate})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0o600)
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func TestUpdateTopologyRefusal(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest := filepath.Join(root, "release.json")
|
||||
config := filepath.Join(root, "config.toml")
|
||||
writeUpdateFixture(t, manifest, `{"required_topology":{"units":["archie-state-store","archie-gateway","archied"],"config_sections":["services.state","services.gateway"]}}`)
|
||||
writeUpdateFixture(t, config, "[services.state]\ntarget='localhost:9090'\n")
|
||||
writeUpdateFixture(t, filepath.Join(root, "systemctl"), "#!/bin/sh\nprintf 'archied.service enabled enabled\\n'\n")
|
||||
t.Setenv("PATH", root+":"+os.Getenv("PATH"))
|
||||
var output bytes.Buffer
|
||||
if err := checkUpdateTopology(config, manifest, &output); err == nil {
|
||||
t.Fatal("partial topology accepted")
|
||||
}
|
||||
for _, missing := range []string{"configure [services.gateway]", "install and enable archie-state-store.service", "install and enable archie-gateway.service"} {
|
||||
if !strings.Contains(output.String(), missing) {
|
||||
t.Fatalf("missing step %q: %s", missing, output.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumeUpdate(t *testing.T) {
|
||||
for _, active := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "interrupted", true: "live"}[active], func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
t.Setenv("ARCHIE_UPDATE_STATE_DIR", root)
|
||||
if err := os.Mkdir(filepath.Join(root, "transaction"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
called := filepath.Join(root, "called")
|
||||
writeUpdateFixture(t, filepath.Join(root, "systemd-run"), "#!/bin/sh\nprintf '%s\\n' \"$@\" > '"+called+"'\n")
|
||||
t.Setenv("PATH", root+":"+os.Getenv("PATH"))
|
||||
lock, err := os.OpenFile(filepath.Join(root, "lock"), os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lock.Close()
|
||||
if active {
|
||||
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
resumed, err := resumeUpdate(t.Context())
|
||||
if err != nil || resumed == active {
|
||||
t.Fatalf("resumed=%v err=%v", resumed, err)
|
||||
}
|
||||
if !active {
|
||||
data, err := os.ReadFile(called)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), "--boot-recover") {
|
||||
t.Fatalf("not a rollback: %s", data)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func writeUpdateFixture(t *testing.T, path, data string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(data), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -944,7 +944,7 @@ type taskDispatcher struct {
|
||||
active int
|
||||
slotsFree *sync.Cond
|
||||
repoTail map[string]chan struct{}
|
||||
wg sync.WaitGroup
|
||||
pending int
|
||||
}
|
||||
|
||||
// SetMaxConcurrency accepts a new global limit without a restart. Running
|
||||
@@ -1004,7 +1004,16 @@ func (d *taskDispatcher) Submit(
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
d.wg.Go(func() {
|
||||
d.mu.Lock()
|
||||
d.pending++
|
||||
d.mu.Unlock()
|
||||
go func() {
|
||||
defer func() {
|
||||
d.mu.Lock()
|
||||
d.pending--
|
||||
d.mu.Unlock()
|
||||
d.slotsFree.Broadcast()
|
||||
}()
|
||||
if previous != nil {
|
||||
<-previous
|
||||
}
|
||||
@@ -1021,11 +1030,15 @@ func (d *taskDispatcher) Submit(
|
||||
}()
|
||||
}
|
||||
process(ctx, task)
|
||||
})
|
||||
}()
|
||||
}
|
||||
|
||||
func (d *taskDispatcher) Wait() {
|
||||
d.wg.Wait()
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
for d.pending > 0 {
|
||||
d.slotsFree.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Daemon) poll(ctx context.Context) {
|
||||
@@ -2214,3 +2227,12 @@ func (d *Daemon) RemoveWorktree(owner, repo, identity string, issue int) error {
|
||||
}
|
||||
return trees.Cleanup(owner, repo, issue)
|
||||
}
|
||||
|
||||
// InFlight counts the same submitted work that graceful shutdown waits for,
|
||||
// including tasks waiting for a repository or concurrency slot.
|
||||
func (d *Daemon) InFlight() int {
|
||||
dispatcher := d.taskDispatcher()
|
||||
dispatcher.mu.Lock()
|
||||
defer dispatcher.mu.Unlock()
|
||||
return dispatcher.pending
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
)
|
||||
|
||||
func TestDispatcherWaitIncludesQueuedWork(t *testing.T) {
|
||||
dispatcher := newTaskDispatcher(1, nil)
|
||||
started := make(chan struct{}, 2)
|
||||
release := make(chan struct{})
|
||||
process := func() { started <- struct{}{}; <-release }
|
||||
for range 2 {
|
||||
dispatcher.Submit(t.Context(), &workflow.Task{Owner: "o", Repo: "r"}, func(_ context.Context, _ *workflow.Task) { process() })
|
||||
}
|
||||
<-started
|
||||
dispatcher.mu.Lock()
|
||||
count := dispatcher.pending
|
||||
dispatcher.mu.Unlock()
|
||||
if count != 2 {
|
||||
t.Fatalf("in-flight count=%d, want running and queued task", count)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() { dispatcher.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("drain completed with pending work")
|
||||
default:
|
||||
}
|
||||
close(release)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("drain did not finish")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/mod/semver"
|
||||
)
|
||||
|
||||
// SelectRelease compares release versions, never git-describe strings. Exact
|
||||
// pins still have to exist in the catalog; a pin cannot invent a release.
|
||||
func SelectRelease(tags []string, channel, pin string) (string, error) {
|
||||
if channel == "" {
|
||||
channel = "stable"
|
||||
}
|
||||
if channel != "stable" && channel != "next" && channel != "exact-pin" {
|
||||
return "", fmt.Errorf("unknown release channel %q", channel)
|
||||
}
|
||||
pin = strings.TrimPrefix(pin, "v")
|
||||
if channel == "exact-pin" && !semver.IsValid("v"+pin) {
|
||||
return "", fmt.Errorf("exact-pin requires a semantic version")
|
||||
}
|
||||
fullVersion := regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+([+-].*)?$`)
|
||||
newest := ""
|
||||
for _, tag := range tags {
|
||||
tag = strings.TrimPrefix(tag, "refs/tags/")
|
||||
tag = strings.TrimPrefix(tag, "archied/")
|
||||
if !fullVersion.MatchString(tag) || !semver.IsValid(tag) {
|
||||
continue
|
||||
}
|
||||
if !releaseMatchesChannel(tag, channel, pin) {
|
||||
continue
|
||||
}
|
||||
if semver.Compare(tag, newest) > 0 {
|
||||
newest = tag
|
||||
}
|
||||
}
|
||||
if newest == "" {
|
||||
return "", fmt.Errorf("no release found for channel %s (pin %s)", channel, pin)
|
||||
}
|
||||
return strings.TrimPrefix(newest, "v"), nil
|
||||
}
|
||||
|
||||
func releaseMatchesChannel(tag, channel, pin string) bool {
|
||||
switch channel {
|
||||
case "stable":
|
||||
return semver.Prerelease(tag) == ""
|
||||
case "exact-pin":
|
||||
return tag == "v"+pin
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package releaseupdate
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSelectRelease(t *testing.T) {
|
||||
tags := []string{"archied/v1.9.0", "v1.10.0", "archied/v2.0.0-rc.2", "v2.0.0-rc.10", "archie/v9.0.0", "nonsense"}
|
||||
for _, tc := range []struct{ channel, pin, want string }{
|
||||
{"stable", "", "1.10.0"},
|
||||
{"", "", "1.10.0"},
|
||||
{"next", "", "2.0.0-rc.10"},
|
||||
{"exact-pin", "1.9.0", "1.9.0"},
|
||||
{"exact-pin", "2.0.0-rc.2", "2.0.0-rc.2"},
|
||||
{"exact-pin", "9.0.0", ""},
|
||||
{"exact-pin", "", ""},
|
||||
{"invalid", "", ""},
|
||||
} {
|
||||
t.Run(tc.channel+tc.pin, func(t *testing.T) {
|
||||
got, err := SelectRelease(tags, tc.channel, tc.pin)
|
||||
if got != tc.want || (err != nil) != (tc.want == "") {
|
||||
t.Fatalf("got %q, %v; want %q", got, err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -19,13 +19,21 @@ const updateResultSentinel = "ARCHIE_UPDATE_RESULT "
|
||||
// CommandCatalog reads a Snapshot JSON document from an explicitly configured
|
||||
// argv command. It never invokes a shell; deployment tooling remains an
|
||||
// administrator-owned adapter.
|
||||
type CommandCatalog struct{ Command []string }
|
||||
type CommandCatalog struct {
|
||||
Command []string
|
||||
ReleaseChannel string
|
||||
Pin string
|
||||
}
|
||||
|
||||
func (c CommandCatalog) Check(ctx context.Context) (Snapshot, error) {
|
||||
if len(c.Command) == 0 {
|
||||
return Snapshot{}, fmt.Errorf("update check command is empty")
|
||||
}
|
||||
output, err := exec.CommandContext(ctx, c.Command[0], c.Command[1:]...).Output()
|
||||
cmd := exec.CommandContext(ctx, c.Command[0], c.Command[1:]...)
|
||||
if c.ReleaseChannel != "" {
|
||||
cmd.Env = append(cmd.Environ(), "ARCHIE_UPDATE_RELEASE_CHANNEL="+c.ReleaseChannel, "ARCHIE_UPDATE_PIN="+c.Pin)
|
||||
}
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return Snapshot{}, fmt.Errorf("run update check: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func TestUpdateTransaction(t *testing.T) {
|
||||
for _, scenario := range []string{"healthy", "checksum", "unsafe-path", "missing-digest", "busy", "lock", "unhealthy", "crash"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
f := newUpdateFixture(t, scenario)
|
||||
var lock *os.File
|
||||
if scenario == "lock" {
|
||||
var err error
|
||||
lock, err = os.OpenFile(filepath.Join(f.state, "lock"), os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lock.Close()
|
||||
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixtureWrite(t, filepath.Join(f.state, "holder"), "fixture-owner", 0o600)
|
||||
}
|
||||
args := []string{}
|
||||
if scenario == "busy" {
|
||||
args = append(args, "--auto")
|
||||
}
|
||||
output, err := f.run(t, f.installer, args...)
|
||||
refusal := map[string]string{"checksum": "checksum mismatch", "unsafe-path": "unsafe archive path", "missing-digest": "digest-pinned", "busy": "work in flight", "lock": "fixture-owner"}[scenario]
|
||||
if refusal != "" {
|
||||
if err == nil || !strings.Contains(output, refusal) {
|
||||
t.Fatalf("wanted %q refusal; err=%v output=%s", refusal, err, output)
|
||||
}
|
||||
f.assertVersion(t, "old")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("installer: %v\n%s", err, output)
|
||||
}
|
||||
f.assertVersion(t, "old") // Staging cannot mutate a running executable.
|
||||
old, err := os.Open(filepath.Join(f.bin, "archied"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer old.Close()
|
||||
watchdog := filepath.Join(f.state, "transaction", "watchdog")
|
||||
output, err = f.run(t, watchdog)
|
||||
if scenario == "crash" {
|
||||
if err == nil {
|
||||
t.Fatal("expected injected crash")
|
||||
}
|
||||
f.env = append(f.env, "CRASH_AFTER=")
|
||||
output, err = f.run(t, watchdog, "--boot-recover")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("watchdog: %v\n%s", err, output)
|
||||
}
|
||||
want := "old"
|
||||
if scenario == "healthy" {
|
||||
want = "new"
|
||||
}
|
||||
f.assertVersion(t, want)
|
||||
data := make([]byte, 100)
|
||||
n, _ := old.Read(data)
|
||||
if !strings.Contains(string(data[:n]), "old") {
|
||||
t.Fatal("promotion overwrote the open old inode")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(f.state, "transaction")); !os.IsNotExist(err) {
|
||||
t.Fatalf("journal not retired: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type updateFixture struct {
|
||||
bin, state, installer string
|
||||
env []string
|
||||
}
|
||||
|
||||
func fixtureWrite(t *testing.T, path, data string, mode os.FileMode) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(data), mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func newUpdateFixture(t *testing.T, scenario string) updateFixture {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
bin, state, fake := filepath.Join(root, "bin"), filepath.Join(root, "state"), filepath.Join(root, "fake")
|
||||
for _, dir := range []string{bin, state, fake} {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
scripts, err := filepath.Abs("../../scripts")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"archied", "archie-state-store", "archie-gateway", "archie-ui", "archie-messaging"} {
|
||||
fixtureWrite(t, filepath.Join(bin, name), "#!/bin/bash\n# old\nexit 0\n", 0o755)
|
||||
}
|
||||
config := filepath.Join(root, "config.toml")
|
||||
fixtureWrite(t, config, "[services.state]\n[services.gateway]\n", 0o600)
|
||||
archive := makeUpdateArchive(t, root, scripts, scenario)
|
||||
sum := sha256.Sum256(archive)
|
||||
if scenario == "checksum" {
|
||||
sum[0]++
|
||||
}
|
||||
fixtureWrite(t, filepath.Join(root, "SHA256SUMS"), fmt.Sprintf("%x archie-core-archied-v2.0.0-linux-amd64.zip\n", sum), 0o600)
|
||||
fixtureWrite(t, filepath.Join(fake, "curl"), `#!/bin/bash
|
||||
out=""
|
||||
for ((i=1;i<=$#;i++)); do
|
||||
if [ "${!i}" = -o ]; then j=$((i+1)); out="${!j}"; fi
|
||||
case "${!i}" in
|
||||
*/SHA256SUMS) source="$FIXTURE/SHA256SUMS" ;;
|
||||
*.zip) source="$FIXTURE/archive.zip" ;;
|
||||
esac
|
||||
done
|
||||
if [ -n "$out" ]; then cp "$source" "$out"; exit; fi
|
||||
if [ "$SCENARIO" = unhealthy ]; then exit 22; fi
|
||||
busy=0
|
||||
[ "$SCENARIO" = busy ] && busy=1
|
||||
printf '{"status":"ok","version":"2.0.0","in_flight":%s}\n' "$busy"
|
||||
`, 0o755)
|
||||
for _, name := range []string{"systemctl", "systemd-run", "docker"} {
|
||||
fixtureWrite(t, filepath.Join(fake, name), "#!/bin/bash\nexit 0\n", 0o755)
|
||||
}
|
||||
fixtureWrite(t, filepath.Join(fake, "git"), "#!/bin/bash\necho unexpected-git >&2\nexit 98\n", 0o755)
|
||||
fixtureWrite(t, filepath.Join(fake, "mv"), `#!/bin/bash
|
||||
/bin/mv "$@" || exit
|
||||
if [ -n "$CRASH_AFTER" ] && [[ "$*" == *".archied.new"* ]]; then kill -KILL "$PPID"; fi
|
||||
`, 0o755)
|
||||
return updateFixture{bin: bin, state: state, installer: filepath.Join(scripts, "archie-update-install"), env: append(os.Environ(),
|
||||
"PATH="+fake+":"+os.Getenv("PATH"), "FIXTURE="+root, "SCENARIO="+scenario,
|
||||
"ARCHIE_BIN_DIR="+bin, "ARCHIE_CONFIG_PATH="+config, "ARCHIE_UPDATE_STATE_DIR="+state,
|
||||
"ARCHIE_UPDATE_DAEMON_PREVIOUS=1.0.0", "ARCHIE_UPDATE_DAEMON_VERSION=2.0.0",
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS=1.0.0", "ARCHIE_UPDATE_AGENT_VERSION="+map[bool]string{true: "2.0.0"}[scenario == "missing-digest"],
|
||||
"ARCHIE_UPDATE_HEALTH_TIMEOUT=1", "ARCHIE_UPDATE_OBSERVE_SECONDS=0", "ARCHIE_UPDATE_POLL_INTERVAL=0.01",
|
||||
"ARCHIE_UPDATE_STATE_STORE_TARGET=", "CRASH_AFTER="+map[bool]string{true: "archied"}[scenario == "crash"],
|
||||
)}
|
||||
}
|
||||
|
||||
func makeUpdateArchive(t *testing.T, root, scripts, scenario string) []byte {
|
||||
t.Helper()
|
||||
file, err := os.Create(filepath.Join(root, "archive.zip"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := zip.NewWriter(file)
|
||||
files := map[string]string{"release.json": `{"required_topology":{"units":["archie-state-store","archie-gateway","archie-ui","archie-messaging","archied"],"config_sections":["services.state","services.gateway"]},"agent_image":"mutable:latest"}`}
|
||||
for _, name := range []string{"archied", "archie-state-store", "archie-gateway", "archie-ui", "archie-messaging"} {
|
||||
files[name] = "#!/bin/bash\n# new\nif [ \"$1\" = -version ]; then echo '" + name + " 2.0.0'; fi\nexit 0\n"
|
||||
}
|
||||
for _, name := range []string{"archie-update-install", "archie-update-watchdog", "archie-update-check"} {
|
||||
data, err := os.ReadFile(filepath.Join(scripts, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files[name] = string(data)
|
||||
}
|
||||
if scenario == "unsafe-path" {
|
||||
files["../escape"] = "escape"
|
||||
}
|
||||
for name, body := range files {
|
||||
header := &zip.FileHeader{Name: "archie-core-archied-v2.0.0-linux-amd64/" + name, Method: zip.Deflate}
|
||||
header.SetMode(0o755)
|
||||
w, err := z.CreateHeader(header)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write([]byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := z.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(file.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func (f updateFixture) run(t *testing.T, path string, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
cmd := exec.CommandContext(t.Context(), path, args...)
|
||||
cmd.Env = f.env
|
||||
data, err := cmd.CombinedOutput()
|
||||
return string(data), err
|
||||
}
|
||||
|
||||
func (f updateFixture) assertVersion(t *testing.T, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(f.bin, "archied"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), "# "+want) {
|
||||
t.Fatalf("wanted %s executable: %s", want, data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package releaseupdate
|
||||
|
||||
import "fmt"
|
||||
|
||||
type Topology struct {
|
||||
Units []string `json:"units"`
|
||||
ConfigSections []string `json:"config_sections"`
|
||||
}
|
||||
|
||||
// MigrationPlan preserves the release's dependency order. Unknown observations
|
||||
// require every step to be checked manually rather than assuming a partial host.
|
||||
func (t Topology) MigrationPlan(units, sections map[string]bool) []string {
|
||||
var steps []string
|
||||
for _, section := range t.ConfigSections {
|
||||
if !sections[section] {
|
||||
steps = append(steps, fmt.Sprintf("configure [%s]", section))
|
||||
}
|
||||
}
|
||||
for _, unit := range t.Units {
|
||||
if !units[unit] {
|
||||
steps = append(steps, fmt.Sprintf("install and enable %s.service", unit))
|
||||
}
|
||||
}
|
||||
return steps
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMigrationPlan(t *testing.T) {
|
||||
topology := Topology{Units: []string{"state", "gateway", "ui", "daemon"}, ConfigSections: []string{"services.state"}}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
units, sections map[string]bool
|
||||
want []string
|
||||
}{
|
||||
{"unknown", nil, nil, []string{"configure [services.state]", "install and enable state.service", "install and enable gateway.service", "install and enable ui.service", "install and enable daemon.service"}},
|
||||
{"missing", map[string]bool{"state": true, "daemon": true}, map[string]bool{"services.state": true}, []string{"install and enable gateway.service", "install and enable ui.service"}},
|
||||
{"complete", map[string]bool{"state": true, "gateway": true, "ui": true, "daemon": true}, map[string]bool{"services.state": true}, nil},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := topology.MigrationPlan(tc.units, tc.sections); !reflect.DeepEqual(got, tc.want) {
|
||||
t.Fatalf("got %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"required_topology": {
|
||||
"units": ["archie-state-store", "archie-gateway", "archie-ui", "archie-messaging", "archied"],
|
||||
"config_sections": ["services.state", "services.gateway"]
|
||||
}
|
||||
}
|
||||
+25
-27
@@ -46,7 +46,7 @@ readonly CURL_TIMEOUT="${ARCHIE_FORGE_TIMEOUT:-20}"
|
||||
readonly TOKEN="${ARCHIE_FORGE_TOKEN:-}"
|
||||
|
||||
# base 1.7.0-4-g0736708 -> 1.7.0
|
||||
base() { printf '%s\n' "${1%%-*}"; }
|
||||
base() { printf '%s\n' "$1" | sed -E 's/-[0-9]+-g[0-9a-f]+$//'; }
|
||||
|
||||
# installed_for reads the version this binary reports for a named component.
|
||||
# archied stamps both its own version and the runtime version the release was
|
||||
@@ -72,35 +72,33 @@ curl_api() {
|
||||
fi
|
||||
}
|
||||
|
||||
# newest_tag_git resolves the newest tag for a component prefix from the git
|
||||
# remote. No local checkout, no working-tree state, no fetch -- `ls-remote`
|
||||
# queries the remote directly, so this works on a host that has never cloned
|
||||
# anything.
|
||||
RELEASE_CHANNEL="${ARCHIE_UPDATE_RELEASE_CHANNEL:-stable}"
|
||||
PIN="${ARCHIE_UPDATE_PIN:-}"
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--channel) RELEASE_CHANNEL="$2"; shift 2 ;;
|
||||
--pin) PIN="$2"; shift 2 ;;
|
||||
*) echo "unknown option: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
select_release() { "$BIN" select-release -channel "$RELEASE_CHANNEL" -pin "$PIN"; }
|
||||
newest_tag_git() {
|
||||
local component="$1" tags
|
||||
command -v git >/dev/null 2>&1 || return 1
|
||||
tags="$(git ls-remote --tags --refs "$SOURCE_URL" "$component/v*" 2>/dev/null)" || return 1
|
||||
[ -n "$tags" ] || return 1
|
||||
printf '%s\n' "$tags" | sed 's|.*refs/tags/||' | sed "s|^$component/v||" | sort -V | tail -1
|
||||
local tags
|
||||
tags="$(git ls-remote --tags --refs "$SOURCE_URL" 'archied/v*' 'v*' 2>/dev/null)" || return 1
|
||||
printf '%s\n' "$tags" | awk '{print $2}' | select_release
|
||||
}
|
||||
|
||||
# newest_tag_api is the API equivalent, used when git is unavailable.
|
||||
newest_tag_api() {
|
||||
local component="$1" page payload found total
|
||||
for page in 1 2 3; do
|
||||
payload="$(curl_api "${API_BASE}/repos/${REPO_SLUG}/tags?per_page=100&page=${page}")" || return 1
|
||||
[ -n "$payload" ] || return 1
|
||||
found="$(printf '%s\n' "$payload" \
|
||||
| grep -o "\"name\":[[:space:]]*\"${component}/v[^\"]*\"" \
|
||||
| sed 's|.*"'"${component}"'/v||; s|"$||' | sort -V | tail -1)"
|
||||
if [ -n "$found" ]; then
|
||||
printf '%s\n' "$found"
|
||||
return 0
|
||||
fi
|
||||
total="$(printf '%s\n' "$payload" | grep -c '"name"')"
|
||||
[ "${total:-0}" -lt 100 ] && break
|
||||
done
|
||||
return 1
|
||||
local page payload tags=""
|
||||
for page in $(seq 1 100); do
|
||||
payload="$(curl_api "${API_BASE}/repos/${REPO_SLUG}/tags?per_page=100&page=${page}")" || return 1
|
||||
tags+="$(jq -er '.[].name' <<< "$payload")"$'\n'
|
||||
if [ "$(jq 'length' <<< "$payload")" -lt 100 ]; then
|
||||
printf '%s' "$tags" | select_release
|
||||
return $?
|
||||
fi
|
||||
done
|
||||
echo "tag catalog exceeded pagination limit" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# artifact_state reports whether a tagged release is PUBLISHED with assets,
|
||||
|
||||
+135
-151
@@ -16,34 +16,8 @@ BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
|
||||
CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}"
|
||||
export PATH="/usr/local/go/bin:$PATH"
|
||||
|
||||
configured_agent_image() {
|
||||
local image
|
||||
image="$(awk '
|
||||
/^\[containers\][[:space:]]*(#.*)?$/ { in_containers = 1; next }
|
||||
/^\[/ { in_containers = 0 }
|
||||
in_containers && /^[[:space:]]*image[[:space:]]*=/ {
|
||||
value = $0
|
||||
sub(/^[[:space:]]*image[[:space:]]*=[[:space:]]*/, "", value)
|
||||
sub(/[[:space:]]*(#.*)?$/, "", value)
|
||||
if (value ~ /^".*"$/) {
|
||||
sub(/^"/, "", value)
|
||||
sub(/"$/, "", value)
|
||||
print value
|
||||
} else if (length(value) >= 2 && substr(value, 1, 1) == sprintf("%c", 39) && substr(value, length(value), 1) == sprintf("%c", 39)) {
|
||||
print substr(value, 2, length(value) - 2)
|
||||
}
|
||||
exit
|
||||
}
|
||||
' "$CONFIG_PATH" 2>/dev/null || true)"
|
||||
printf '%s' "$image"
|
||||
}
|
||||
|
||||
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-$(configured_agent_image)}"
|
||||
AGENT_IMAGE="${AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
|
||||
AGENT_IMAGE_BACKUP="$BIN_DIR/archie-agent-image.prev.tar"
|
||||
# State for the managed image, independent of archied's daemon build stamp.
|
||||
AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
|
||||
AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}"
|
||||
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
|
||||
# The address the daemon dials for its state, so the watchdog can wait for it to
|
||||
# accept connections before it starts archied. Read with a narrow awk: the
|
||||
@@ -67,9 +41,27 @@ configured_state_store_target() {
|
||||
|
||||
STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-$(configured_state_store_target)}"
|
||||
|
||||
GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui archie-messaging"
|
||||
GATEWAY_SERVICES="archie-state-store archie-gateway archie-ui archie-messaging archied"
|
||||
GATEWAY_BINARIES="$GATEWAY_SERVICES"
|
||||
|
||||
SOURCE_BUILD="${ARCHIE_UPDATE_SOURCE_BUILD:-0}"
|
||||
AUTO=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--from-source) SOURCE_BUILD=1 ;;
|
||||
--auto) AUTO=1 ;;
|
||||
*) echo "unknown option: $arg" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$AUTO" = 1 ] && [ -z "${ARCHIE_UPDATE_DAEMON_VERSION:-}${ARCHIE_UPDATE_AGENT_VERSION:-}" ]; then
|
||||
snapshot="$("$(dirname "$0")/archie-update-check")"
|
||||
export ARCHIE_UPDATE_DAEMON_PREVIOUS="$(jq -r '.Components[] | select(.ID == "daemon") | .Installed' <<< "$snapshot")"
|
||||
export ARCHIE_UPDATE_DAEMON_VERSION="$(jq -r '.Components[] | select(.ID == "daemon") | .Available' <<< "$snapshot")"
|
||||
export ARCHIE_UPDATE_AGENT_PREVIOUS="$(jq -r '.Components[] | select(.ID == "agent") | .Installed' <<< "$snapshot")"
|
||||
export ARCHIE_UPDATE_AGENT_VERSION="$(jq -r '.Components[] | select(.ID == "agent") | .Available' <<< "$snapshot")"
|
||||
fi
|
||||
|
||||
PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}"
|
||||
PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}"
|
||||
GW_TARGET="${ARCHIE_UPDATE_DAEMON_VERSION:-}"
|
||||
@@ -132,146 +124,138 @@ for target in "$GW_TARGET" "$RT_TARGET"; do
|
||||
fi
|
||||
done
|
||||
|
||||
STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}"
|
||||
mkdir -p "$STATE_DIR"
|
||||
chmod 700 "$STATE_DIR"
|
||||
exec 9>"$STATE_DIR/lock"
|
||||
if ! flock -n 9; then
|
||||
echo "update locked by $(cat "$STATE_DIR/holder" 2>/dev/null); journal: $STATE_DIR/transaction" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'installer PID %s\n' "$$" > "$STATE_DIR/holder"
|
||||
if [ -d "$STATE_DIR/transaction" ]; then
|
||||
echo "unfinished update: run $STATE_DIR/transaction/watchdog --recover before retrying" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tmp="$(mktemp -d)"
|
||||
source_dir="$tmp/source"
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
|
||||
echo "==> fetching approved component releases"
|
||||
|
||||
git clone --quiet --filter=blob:none --no-checkout "$SOURCE_URL" "$source_dir"
|
||||
git -C "$source_dir" fetch --quiet --tags origin
|
||||
|
||||
gw_commit=""
|
||||
rt_commit=""
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
gw_commit="$(git -C "$source_dir" rev-parse --verify "refs/tags/archied/v$GW^{commit}")"
|
||||
source_dir="$tmp/source"
|
||||
release_dir="$tmp/release"
|
||||
mkdir -p "$release_dir"
|
||||
release_version="${GW_TARGET:-$RT_TARGET}"
|
||||
if [ -n "$GW_TARGET" ] && [ -n "$RT_TARGET" ] && [ "$GW_TARGET" != "$RT_TARGET" ]; then
|
||||
echo "daemon and agent must name the same release" >&2; exit 1
|
||||
fi
|
||||
if [ "$SOURCE_BUILD" = 1 ]; then
|
||||
tag="v$release_version"
|
||||
git ls-remote --exit-code --tags "$SOURCE_URL" "refs/tags/$tag" >/dev/null || tag="archied/v$release_version"
|
||||
git clone --quiet --depth=1 --branch "$tag" "$SOURCE_URL" "$source_dir"
|
||||
cp "$source_dir/release.json" "$release_dir/"
|
||||
GATEWAY_BINARIES="$(jq -er '.required_topology.units | join(" ")' "$release_dir/release.json")"
|
||||
cp "$source_dir"/scripts/archie-update-* "$release_dir/"
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
for cmd in $GATEWAY_BINARIES; do
|
||||
[[ "$cmd" =~ ^archie[a-z0-9-]*$ ]] || { echo "invalid release unit: $cmd" >&2; exit 1; }
|
||||
(cd "$source_dir" && go build -ldflags "-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$GW -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$RT" -o "$release_dir/$cmd" "./cmd/$cmd")
|
||||
done
|
||||
fi
|
||||
else
|
||||
case "$(uname -s)/$(uname -m)" in
|
||||
Linux/x86_64) platform=linux-amd64 ;;
|
||||
*) echo "no published artifact for $(uname -s)/$(uname -m)" >&2; exit 1 ;;
|
||||
esac
|
||||
name="archie-core-archied-v$release_version-$platform"
|
||||
base="${ARCHIE_RELEASE_BASE_URL:-https://github.com/samcharles93/archie-core/releases/download}/v$release_version"
|
||||
if ! curl -fLSs --max-time 300 "$base/$name.zip" -o "$tmp/$name.zip"; then
|
||||
base="${ARCHIE_RELEASE_BASE_URL:-https://github.com/samcharles93/archie-core/releases/download}/archied/v$release_version"
|
||||
curl -fLSs --max-time 300 "$base/$name.zip" -o "$tmp/$name.zip"
|
||||
fi
|
||||
curl -fLSs --max-time 60 "$base/SHA256SUMS" -o "$tmp/SHA256SUMS"
|
||||
expected="$(awk -v name="$name.zip" '$2 == name {print $1}' "$tmp/SHA256SUMS")"
|
||||
[[ "$expected" =~ ^[a-fA-F0-9]{64}$ ]] || { echo "missing or ambiguous archive checksum" >&2; exit 1; }
|
||||
printf '%s %s\n' "$expected" "$tmp/$name.zip" | sha256sum --check --status || { echo "release checksum mismatch" >&2; exit 1; }
|
||||
# Reject paths outside the single distribution directory before extracting.
|
||||
while IFS= read -r entry; do
|
||||
case "$entry" in "$name/"*) ;; *) echo "unsafe archive path: $entry" >&2; exit 1 ;; esac
|
||||
case "/$entry/" in */../*|*/./*|*\\*) echo "unsafe archive path: $entry" >&2; exit 1 ;; esac
|
||||
done < <(unzip -Z1 "$tmp/$name.zip")
|
||||
if unzip -Z -l "$tmp/$name.zip" | awk '$1 ~ /^l/ {found=1} END {exit !found}'; then
|
||||
echo "release contains symlinks" >&2; exit 1
|
||||
fi
|
||||
unzip -q "$tmp/$name.zip" -d "$tmp/unpacked"
|
||||
if find "$tmp/unpacked" -type l | grep -q .; then echo "release contains symlinks" >&2; exit 1; fi
|
||||
cp -a "$tmp/unpacked/$name/." "$release_dir/"
|
||||
fi
|
||||
|
||||
# Observation and config parsing belong to the daemon, including on old hosts.
|
||||
observer="$release_dir/archied"
|
||||
[ -x "$observer" ] || observer="$BIN_DIR/archied"
|
||||
"$observer" update-topology -config "$CONFIG_PATH" -release "$release_dir/release.json"
|
||||
GATEWAY_SERVICES="$(jq -er '.required_topology.units | join(" ")' "$release_dir/release.json")"
|
||||
GATEWAY_BINARIES="$GATEWAY_SERVICES"
|
||||
|
||||
if [[ ",$components," == *,agent,* ]]; then
|
||||
rt_commit="$(git -C "$source_dir" rev-parse --verify "refs/tags/archied/v$RT^{commit}")"
|
||||
fi
|
||||
build_commit="$gw_commit"
|
||||
[ -n "$build_commit" ] || build_commit="$rt_commit"
|
||||
git -C "$source_dir" checkout --quiet --detach "$build_commit"
|
||||
cd "$source_dir"
|
||||
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
missing_units=""
|
||||
for svc in $GATEWAY_SERVICES; do
|
||||
[ -d "$source_dir/cmd/$svc" ] || continue
|
||||
if ! systemctl --user list-unit-files "$svc.service" --no-legend 2>/dev/null | grep -q .; then
|
||||
missing_units="${missing_units:+$missing_units }$svc.service"
|
||||
fi
|
||||
done
|
||||
if [ -n "$missing_units" ]; then
|
||||
echo "refusing to install archied $GW: it runs as separate processes and this host has no unit for: $missing_units" >&2
|
||||
echo "create them from deployments/systemd-user-service.md, then retry the update" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
unmanaged_units=""
|
||||
for dir in "$source_dir"/cmd/*/; do
|
||||
[ -d "$dir" ] || continue
|
||||
svc="$(basename "$dir")"
|
||||
case " $GATEWAY_SERVICES " in
|
||||
*" $svc "*) continue ;;
|
||||
esac
|
||||
if systemctl --user list-unit-files "$svc.service" --no-legend 2>/dev/null | grep -q .; then
|
||||
unmanaged_units="${unmanaged_units:+$unmanaged_units }$svc.service"
|
||||
fi
|
||||
done
|
||||
if [ -n "$unmanaged_units" ]; then
|
||||
echo "refusing to install archied $GW: this host runs units the update does not manage: $unmanaged_units" >&2
|
||||
echo "a current install.sh writes units for: $GATEWAY_SERVICES. If the installed updater predates this release, replace $BIN_DIR/archie-update-install with scripts/archie-update-install from archied/v$GW; otherwise remove the extra unit, or add it to GATEWAY_SERVICES if this host is meant to run it." >&2
|
||||
exit 1
|
||||
if [ "$SOURCE_BUILD" = 1 ]; then
|
||||
docker build --build-arg "RUNTIME_VERSION=$RT" --iidfile "$tmp/image-id" "$source_dir"
|
||||
AGENT_IMAGE="$(cat "$tmp/image-id")"
|
||||
else
|
||||
AGENT_IMAGE="$(jq -er '.agent_image' "$release_dir/release.json")"
|
||||
[[ "$AGENT_IMAGE" =~ @sha256:[a-f0-9]{64}$ ]] || { echo "refusing agent update: release has no digest-pinned image" >&2; exit 1; }
|
||||
docker pull "$AGENT_IMAGE"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
echo "==> snapshotting the database before the update migrates it"
|
||||
mkdir -p "$(dirname "$DB_SNAPSHOT")"
|
||||
if ! go run ./cmd/archie-state-store backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT"; then
|
||||
echo "refusing to install $GW: could not snapshot the database named by $CONFIG_PATH, so a failed migration could not be recovered" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$AUTO" = 1 ]; then
|
||||
status="$(curl -fsS --max-time 10 "${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}/health/detailed")"
|
||||
[ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || { echo "refusing auto-update: daemon has work in flight or cannot report it" >&2; exit 1; }
|
||||
fi
|
||||
|
||||
install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install"
|
||||
install -m755 "$source_dir/scripts/archie-update-watchdog" "$BIN_DIR/archie-update-watchdog"
|
||||
install -m755 "$source_dir/scripts/archie-update-check" "$BIN_DIR/archie-update-check"
|
||||
|
||||
built=""
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary"
|
||||
ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$GW"
|
||||
ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$RT"
|
||||
for cmd in $GATEWAY_BINARIES; do
|
||||
[ -d "./cmd/$cmd" ] || continue
|
||||
echo "==> building $cmd $GW"
|
||||
go build -ldflags "$ldflags" -o "$tmp/$cmd" "./cmd/$cmd"
|
||||
built="${built:+$built }$cmd"
|
||||
reported="$("$release_dir/$cmd" -version | awk -v cmd="$cmd" '$1 == cmd {print $2; exit}')"
|
||||
[ "$reported" = "$GW" ] || { echo "candidate $cmd reports $reported, expected $GW" >&2; exit 1; }
|
||||
done
|
||||
|
||||
echo "==> backing up current binaries"
|
||||
for cmd in $built; do
|
||||
if [ -f "$BIN_DIR/$cmd" ]; then
|
||||
cp -f "$BIN_DIR/$cmd" "$BIN_DIR/$cmd.prev"
|
||||
else
|
||||
rm -f "$BIN_DIR/$cmd.prev"
|
||||
fi
|
||||
done
|
||||
echo "==> installing $built"
|
||||
for cmd in $built; do
|
||||
install -m755 "$tmp/$cmd" "$BIN_DIR/$cmd"
|
||||
done
|
||||
|
||||
if ! verify_installed_versions "$GW"; then
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p "$(dirname "$DB_SNAPSHOT")"
|
||||
"$BIN_DIR/archie-state-store" backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT"
|
||||
built="$GATEWAY_BINARIES archie-update-install archie-update-watchdog archie-update-check"
|
||||
fi
|
||||
|
||||
# Freeze recovery code and every prior file before publishing the intent.
|
||||
journal="$STATE_DIR/preparing"
|
||||
rm -rf "$journal"
|
||||
mkdir -p "$journal/previous"
|
||||
cp "$release_dir/archie-update-watchdog" "$journal/watchdog"
|
||||
cp "$observer" "$journal/helper"
|
||||
if [[ ",$components," == *,agent,* ]]; then
|
||||
git checkout --quiet --detach "$rt_commit"
|
||||
echo "==> saving managed worker image $AGENT_IMAGE for rollback"
|
||||
docker image save --output "$AGENT_IMAGE_BACKUP" "$AGENT_IMAGE"
|
||||
if [ -f "$AGENT_VERSION_FILE" ]; then
|
||||
cp -f "$AGENT_VERSION_FILE" "$AGENT_VERSION_BACKUP"
|
||||
AGENT_VERSION_PREVIOUS_EXISTS=1
|
||||
else
|
||||
rm -f "$AGENT_VERSION_BACKUP"
|
||||
AGENT_VERSION_PREVIOUS_EXISTS=0
|
||||
fi
|
||||
echo "==> rebuilding managed worker image $AGENT_IMAGE for runtime $RT"
|
||||
docker build --build-arg "RUNTIME_VERSION=$RT" --tag "$AGENT_IMAGE" .
|
||||
version_tmp="$(mktemp "$AGENT_VERSION_FILE.tmp.XXXXXX")"
|
||||
printf '%s\n' "$RT" > "$version_tmp"
|
||||
chmod 600 "$version_tmp"
|
||||
mv -f "$version_tmp" "$AGENT_VERSION_FILE"
|
||||
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -prepare "$AGENT_IMAGE"
|
||||
fi
|
||||
|
||||
echo "==> handing off to the update watchdog for restart and health verification"
|
||||
chmod 700 "$journal/watchdog"
|
||||
for cmd in $built; do
|
||||
if [ -f "$BIN_DIR/$cmd" ]; then cp -p "$BIN_DIR/$cmd" "$journal/previous/$cmd"; fi
|
||||
install -m755 "$release_dir/$cmd" "$BIN_DIR/.$cmd.new"
|
||||
done
|
||||
if [ -f "$AGENT_VERSION_FILE" ]; then cp -p "$AGENT_VERSION_FILE" "$journal/agent.version"; fi
|
||||
# No unit/config changes are made: topology migration is explicitly refused.
|
||||
export ARCHIE_UPDATE_AUTO="$AUTO"
|
||||
export ARCHIE_UPDATE_STATE_DIR="$STATE_DIR"
|
||||
export ARCHIE_BIN_DIR="$BIN_DIR" ARCHIE_CONFIG_PATH="$CONFIG_PATH"
|
||||
export ARCHIE_UPDATE_COMPONENTS="$components" ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" ARCHIE_UPDATE_BINARIES="$built"
|
||||
export ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT"
|
||||
export ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT"
|
||||
export ARCHIE_AGENT_VERSION_FILE="$AGENT_VERSION_FILE" ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT"
|
||||
export ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET"
|
||||
# Bash's quoted declarations preserve spaces without interpreting journal values.
|
||||
while IFS= read -r var; do declare -p "$var"; done < <(compgen -e | grep -E '^ARCHIE_(UPDATE_|BIN_DIR$|CONFIG_PATH$|HEALTH_URL$|AGENT_VERSION_FILE$|DB_SNAPSHOT$)') > "$journal/environment"
|
||||
printf 'prepared\n' > "$journal/phase"
|
||||
sync -f "$BIN_DIR"
|
||||
sync -f "$journal"
|
||||
mv "$journal" "$STATE_DIR/transaction"
|
||||
sync -f "$STATE_DIR"
|
||||
journal="$STATE_DIR/transaction"
|
||||
# The independent watchdog holds the lock and owns promotion, restart and rollback.
|
||||
systemd-run --user --unit=archied-update-restart --collect --quiet \
|
||||
--setenv=ARCHIE_BIN_DIR="$BIN_DIR" \
|
||||
--setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-}" \
|
||||
--setenv=ARCHIE_UPDATE_REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" \
|
||||
--setenv=ARCHIE_UPDATE_CHANNEL="${ARCHIE_UPDATE_CHANNEL:-}" \
|
||||
--setenv=ARCHIE_UPDATE_CHAT_ID="${ARCHIE_UPDATE_CHAT_ID:-0}" \
|
||||
--setenv=ARCHIE_UPDATE_THREAD_ID="${ARCHIE_UPDATE_THREAD_ID:-0}" \
|
||||
--setenv=ARCHIE_UPDATE_COMPONENTS="$components" \
|
||||
--setenv=ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" \
|
||||
--setenv=ARCHIE_UPDATE_BINARIES="$built" \
|
||||
--setenv=ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" \
|
||||
--setenv=ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT" \
|
||||
--setenv=ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" \
|
||||
--setenv=ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT" \
|
||||
--setenv=ARCHIE_AGENT_IMAGE="$AGENT_IMAGE" \
|
||||
--setenv=ARCHIE_AGENT_IMAGE_BACKUP="$AGENT_IMAGE_BACKUP" \
|
||||
--setenv=ARCHIE_AGENT_VERSION_FILE="$AGENT_VERSION_FILE" \
|
||||
--setenv=ARCHIE_AGENT_VERSION_BACKUP="$AGENT_VERSION_BACKUP" \
|
||||
--setenv=ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS="${AGENT_VERSION_PREVIOUS_EXISTS:-0}" \
|
||||
--setenv=ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT" \
|
||||
--setenv=ARCHIE_CONFIG_PATH="$CONFIG_PATH" \
|
||||
--setenv=ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET" \
|
||||
"$BIN_DIR/archie-update-watchdog"
|
||||
|
||||
--setenv=ARCHIE_UPDATE_STATE_DIR="$STATE_DIR" "$journal/watchdog"
|
||||
case "$components" in
|
||||
daemon)
|
||||
previous="{\"daemon\":\"$(json_escape "$PREV_GW")\"}"
|
||||
|
||||
+116
-134
@@ -7,36 +7,39 @@
|
||||
# It does not rebuild or reinstall the gateway binary.
|
||||
set -uo pipefail
|
||||
|
||||
BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
|
||||
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
|
||||
AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}"
|
||||
AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
|
||||
AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}"
|
||||
AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}"
|
||||
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
|
||||
CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}"
|
||||
# The daemon exports its own health address (releaseupdate.CommandInstaller);
|
||||
# this default only covers a hand-driven run, and must stay the address archied
|
||||
# itself serves /healthz on -- [health] listen, defaulted in
|
||||
# internal/infrastructure/configuration/defaults.go.
|
||||
STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}"
|
||||
journal="$STATE_DIR/transaction"
|
||||
mkdir -p "$STATE_DIR"
|
||||
exec 9>"$STATE_DIR/lock"
|
||||
if [ "${1:-}" = --recover ]; then
|
||||
flock -n 9 || exit 0 # A live installer/watchdog owns recovery.
|
||||
else
|
||||
flock -w 60 9 || { echo "update locked by $(cat "$STATE_DIR/holder")" >&2; exit 1; }
|
||||
fi
|
||||
[ -d "$journal" ] || exit 0
|
||||
printf 'watchdog PID %s\n' "$$" > "$STATE_DIR/holder"
|
||||
source "$journal/environment"
|
||||
BIN_DIR="$ARCHIE_BIN_DIR"
|
||||
CONFIG_PATH="$ARCHIE_CONFIG_PATH"
|
||||
DB_SNAPSHOT="$ARCHIE_DB_SNAPSHOT"
|
||||
AGENT_VERSION_FILE="$ARCHIE_AGENT_VERSION_FILE"
|
||||
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}"
|
||||
HEALTH_URL="${HEALTH_URL%/}"
|
||||
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
|
||||
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
|
||||
POLL_INTERVAL_SECONDS=3
|
||||
COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}"
|
||||
# Set by archie-update-install: the units to restart, in start order, and the
|
||||
# binaries it replaced. The defaults keep a hand-driven run on a pre-
|
||||
# decomposition host working.
|
||||
UNITS="${ARCHIE_UPDATE_UNITS:-archied}"
|
||||
BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}"
|
||||
|
||||
# The State Store's address, so the start order below can mean "ready" rather than
|
||||
# "launched". Empty when the config names none or the installed updater predates
|
||||
# passing one -- waiting is then skipped, which is the previous behaviour.
|
||||
OBSERVE_SECONDS="${ARCHIE_UPDATE_OBSERVE_SECONDS:-120}"
|
||||
POLL_INTERVAL_SECONDS="${ARCHIE_UPDATE_POLL_INTERVAL:-3}"
|
||||
COMPONENTS="$ARCHIE_UPDATE_COMPONENTS"
|
||||
UNITS="$ARCHIE_UPDATE_UNITS"
|
||||
BINARIES="$ARCHIE_UPDATE_BINARIES"
|
||||
STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-}"
|
||||
STATE_STORE_TIMEOUT_SECONDS="${ARCHIE_UPDATE_STATE_STORE_TIMEOUT:-30}"
|
||||
|
||||
phase() {
|
||||
printf '%s\n' "$1" > "$journal/phase.new" && sync -f "$journal" &&
|
||||
mv "$journal/phase.new" "$journal/phase" && sync -f "$journal"
|
||||
}
|
||||
|
||||
json_escape() {
|
||||
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
|
||||
}
|
||||
@@ -75,10 +78,22 @@ EOF
|
||||
mv -f "$tmp" "$REPORT_PATH"
|
||||
}
|
||||
|
||||
healthy() {
|
||||
local report
|
||||
report="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 1
|
||||
[ "$(jq -r '.version' <<< "$report")" = "$ARCHIE_UPDATE_INSTALLED_GATEWAY" ] &&
|
||||
[ "$(jq -r '.status' <<< "$report")" = ok ]
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS))
|
||||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||||
if curl -sf -o /dev/null "$HEALTH_URL/healthz"; then
|
||||
if healthy; then
|
||||
deadline=$((SECONDS + OBSERVE_SECONDS))
|
||||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||||
sleep "$POLL_INTERVAL_SECONDS"
|
||||
healthy || return 1
|
||||
done
|
||||
return 0
|
||||
fi
|
||||
sleep "$POLL_INTERVAL_SECONDS"
|
||||
@@ -86,20 +101,7 @@ wait_healthy() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# manage_unit runs one systemctl action. archied is required: this whole
|
||||
# mechanism exists to restart it, and a failed stop must abort the rollback
|
||||
# rather than let it race a live process over the binary and the database.
|
||||
# The sibling processes are best-effort, because archie-update-install has
|
||||
# already refused the update if a unit for one of them was missing, and a
|
||||
# probe that guesses wrong must not silently skip the required stop.
|
||||
manage_unit() {
|
||||
local action="$1" unit="$2"
|
||||
if [ "$unit" = archied ]; then
|
||||
systemctl --user "$action" "$unit.service"
|
||||
return
|
||||
fi
|
||||
systemctl --user "$action" "$unit.service" || true
|
||||
}
|
||||
manage_unit() { systemctl --user "$1" "$2.service"; }
|
||||
|
||||
# port_accepts reports whether host:port accepts a connection. bash's /dev/tcp
|
||||
# costs no process and no dependency, and accepting is exactly the signal this
|
||||
@@ -109,21 +111,7 @@ port_accepts() {
|
||||
(exec 3<>"/dev/tcp/${1%:*}/${1##*:}") 2>/dev/null
|
||||
}
|
||||
|
||||
# wait_for_state_store blocks until the State Store accepts connections, or the
|
||||
# budget runs out.
|
||||
#
|
||||
# Ordering the units is not enough on its own: `systemctl --user start` returns
|
||||
# once the process is forked, not once its port accepts. Measured on a 1.37.1 ->
|
||||
# 1.38.0 update, the State Store's unit reported started while it was still
|
||||
# binding, archied was started next, dialled a closed port and exited 1, and only
|
||||
# systemd's Restart=on-failure brought it back seven seconds later. A host whose
|
||||
# unit lacks that restart policy gets a failed update behind a healthy-looking
|
||||
# "build and install succeeded" line.
|
||||
#
|
||||
# A timeout does NOT fail the update. If the State Store never comes up, archied
|
||||
# fails closed on its own and wait_healthy rolls the release back -- waiting is an
|
||||
# optimisation for the ordinary case, and losing the race must stay a slower path
|
||||
# to the same place rather than a second failure mode.
|
||||
# Wait for the dependency before starting its callers.
|
||||
wait_for_state_store() {
|
||||
[ -n "$STATE_STORE_TARGET" ] || {
|
||||
echo "==> no State Store target known; starting the rest without waiting for it" >&2
|
||||
@@ -169,39 +157,70 @@ stop_units() {
|
||||
return $rc
|
||||
}
|
||||
|
||||
# rollback_gateway restores every binary the install replaced. A binary with
|
||||
# no .prev was introduced by the failed release, so it is removed rather than
|
||||
# left behind pointing at processes the restored release does not run.
|
||||
rollback_gateway() {
|
||||
local cmd rc=0
|
||||
rollback_files() {
|
||||
local cmd
|
||||
for cmd in $BINARIES; do
|
||||
if [ -f "$BIN_DIR/$cmd.prev" ]; then
|
||||
cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1
|
||||
if [ -f "$journal/previous/$cmd" ]; then
|
||||
cp -p "$journal/previous/$cmd" "$BIN_DIR/.$cmd.rollback" &&
|
||||
mv -f "$BIN_DIR/.$cmd.rollback" "$BIN_DIR/$cmd" || return 1
|
||||
else
|
||||
rm -f "$BIN_DIR/$cmd" || rc=1
|
||||
rm -f "$BIN_DIR/$cmd" || return 1
|
||||
fi
|
||||
rm -f "$BIN_DIR/.$cmd.new"
|
||||
done
|
||||
return $rc
|
||||
if [[ ",$COMPONENTS," == *,agent,* ]]; then
|
||||
if [ -f "$journal/agent.version" ]; then
|
||||
cp -p "$journal/agent.version" "$AGENT_VERSION_FILE.new" &&
|
||||
mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1
|
||||
else
|
||||
rm -f "$AGENT_VERSION_FILE" || return 1
|
||||
fi
|
||||
fi
|
||||
sync -f "$BIN_DIR"
|
||||
}
|
||||
|
||||
rollback_agent_image() {
|
||||
if [ ! -f "$AGENT_IMAGE_BACKUP" ]; then
|
||||
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
|
||||
return 1
|
||||
fi
|
||||
docker image load --input "$AGENT_IMAGE_BACKUP"
|
||||
finish() {
|
||||
phase "$1" || return 1
|
||||
# A terminal journal is harmless if power fails before removal.
|
||||
rm -rf "$STATE_DIR/finished"
|
||||
mv "$journal" "$STATE_DIR/finished" && sync -f "$STATE_DIR" || return 1
|
||||
rm -rf "$STATE_DIR/finished"
|
||||
}
|
||||
|
||||
# The sidecar describes the image restored above. Restore its prior state in
|
||||
# the same rollback path; a first install has no prior file and must remove the
|
||||
# newly-created metadata rather than leave a false installed version behind.
|
||||
rollback_agent_version() {
|
||||
if [ "$AGENT_VERSION_PREVIOUS_EXISTS" = 1 ]; then
|
||||
[ -f "$AGENT_VERSION_BACKUP" ] || return 1
|
||||
cp -f "$AGENT_VERSION_BACKUP" "$AGENT_VERSION_FILE"
|
||||
else
|
||||
rm -f "$AGENT_VERSION_FILE"
|
||||
rollback() {
|
||||
phase rolling-back || return 1
|
||||
stop_units && rollback_files || return 1
|
||||
if [ -f "$journal/image.json" ]; then
|
||||
manage_unit start archie-state-store && wait_for_state_store &&
|
||||
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -rollback || return 1
|
||||
fi
|
||||
phase rolled-back || return 1
|
||||
start_units || return 1
|
||||
write_report failed true || return 1
|
||||
finish rolled-back
|
||||
}
|
||||
|
||||
promote() {
|
||||
if [ "${ARCHIE_UPDATE_AUTO:-0}" = 1 ]; then
|
||||
local status
|
||||
status="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 2
|
||||
[ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || return 2
|
||||
fi
|
||||
phase switching || return 1
|
||||
stop_units || return 1
|
||||
for cmd in $BINARIES; do
|
||||
mv -f "$BIN_DIR/.$cmd.new" "$BIN_DIR/$cmd" || return 1
|
||||
done
|
||||
if [[ ",$COMPONENTS," == *,agent,* ]]; then
|
||||
printf '%s\n' "$ARCHIE_UPDATE_INSTALLED_RUNTIME" > "$AGENT_VERSION_FILE.new" &&
|
||||
mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1
|
||||
fi
|
||||
sync -f "$BIN_DIR" || return 1
|
||||
if [ -f "$journal/image.json" ]; then
|
||||
manage_unit start archie-state-store && wait_for_state_store &&
|
||||
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" || return 1
|
||||
fi
|
||||
phase observing && start_units && wait_healthy
|
||||
}
|
||||
|
||||
# explain_database_recovery states what a daemon rollback does not do. The
|
||||
@@ -214,62 +233,25 @@ explain_database_recovery() {
|
||||
echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2
|
||||
}
|
||||
|
||||
start_units
|
||||
|
||||
if wait_healthy; then
|
||||
echo "==> new version is healthy"
|
||||
write_report "passed" false
|
||||
case "$(cat "$journal/phase")" in
|
||||
committed) start_units && finish committed; exit $? ;;
|
||||
rolled-back) start_units && finish rolled-back; exit $? ;;
|
||||
esac
|
||||
if [ -z "${1:-}" ] && promote; then
|
||||
write_report passed false && finish committed
|
||||
exit $?
|
||||
fi
|
||||
if [ "$(cat "$journal/phase")" = prepared ]; then
|
||||
echo "update was not promoted; leaving running work alone" >&2
|
||||
for cmd in $BINARIES; do rm -f "$BIN_DIR/.$cmd.new"; done
|
||||
write_report failed false && finish rolled-back || exit 1
|
||||
if [ "${1:-}" = --boot-recover ]; then start_units; exit $?; fi
|
||||
exit 1
|
||||
fi
|
||||
echo "==> restoring interrupted or unhealthy update" >&2
|
||||
if rollback; then
|
||||
if [[ ",$COMPONENTS," == *,daemon,* ]]; then explain_database_recovery; fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components"
|
||||
rolled_back=false
|
||||
# Stop before touching the executable. The service uses Restart=on-failure in
|
||||
# the supported unit, so copying rollback state while a crash-loop child is
|
||||
# alive can execute the new binary after its files have been replaced.
|
||||
if stop_units; then
|
||||
case "$COMPONENTS" in
|
||||
daemon)
|
||||
if [ -f "$BIN_DIR/archied.prev" ]; then
|
||||
if rollback_gateway; then
|
||||
rolled_back=true
|
||||
fi
|
||||
else
|
||||
echo "==> no backup for archied; cannot roll it back automatically" >&2
|
||||
fi
|
||||
;;
|
||||
agent)
|
||||
if [ -f "$AGENT_IMAGE_BACKUP" ]; then
|
||||
if rollback_agent_image && rollback_agent_version; then
|
||||
rolled_back=true
|
||||
fi
|
||||
else
|
||||
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
|
||||
fi
|
||||
;;
|
||||
daemon,agent)
|
||||
# Preserve the old all-or-nothing rollback guarantee: never restore one
|
||||
# component and leave the other on the failed release.
|
||||
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ]; then
|
||||
if rollback_agent_image && rollback_agent_version && rollback_gateway; then
|
||||
rolled_back=true
|
||||
fi
|
||||
else
|
||||
echo "==> not all changed-component backups exist; cannot roll back automatically" >&2
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
|
||||
;;
|
||||
esac
|
||||
else
|
||||
echo "==> could not stop every unit; refusing rollback while one may be running" >&2
|
||||
fi
|
||||
if [ "$rolled_back" = true ]; then
|
||||
start_units
|
||||
fi
|
||||
if [[ ",$COMPONENTS," == *,daemon,* ]]; then
|
||||
explain_database_recovery
|
||||
fi
|
||||
|
||||
write_report "failed" "$rolled_back"
|
||||
echo "rollback incomplete; retained journal: $journal" >&2
|
||||
exit 1
|
||||
Reference in new issue
Block a user