feat(releaseupdate): install from release artifacts with journaled rollback, topology check and channels

This commit is contained in:
sam committed 2026-10-05 17:02:29 +11:00
1 parent 9bb20830f3
commit ee2c9cda97
24 files changed
+1186 -333

No files matched your search

+22 -7
View File
@@ -166,6 +166,8 @@ jobs:
fi
build-and-push:
outputs:
agent_digest: ${{ steps.agent.outputs.digest }}
needs: gate
runs-on: ubuntu-latest
steps:
@@ -258,6 +260,7 @@ jobs:
cache-to: type=gha,mode=max,scope=archied
- name: Build and push archie-agent
id: agent
if: steps.versions.outputs.runtime_tag != '' || steps.versions.outputs.release_tag != ''
uses: docker/build-push-action@v6
with:
@@ -282,7 +285,7 @@ jobs:
# the Docker image build-and-push already publishes, never as a host
# binary (archie-core-8cda.4.7.1).
dist-zip:
needs: gate
needs: [gate, build-and-push]
# Not gated on github.ref. A release is pushed as main plus both tags at
# one SHA, the concurrency group below collapses those into a single run,
# and the survivor is whichever ref happened to start last -- for
@@ -329,7 +332,7 @@ jobs:
# list and the copy list below stop matching the commands in cmd/.
ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=${{ steps.version.outputs.version }} -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=${{ steps.version.outputs.version }}"
mkdir -p dist
for cmd in archied archie-gateway archie-state-store archie-ui archie-messaging; do
for cmd in $(jq -r '.required_topology.units[]' release.json); do
go build -ldflags "$ldflags" -o "dist/$cmd" "./cmd/$cmd"
done
@@ -339,19 +342,28 @@ jobs:
set -e
name="archie-core-archied-v${{ steps.version.outputs.version }}-linux-amd64"
mkdir -p "$name"
cp dist/archied dist/archie-gateway dist/archie-state-store dist/archie-ui dist/archie-messaging "$name/"
for cmd in $(jq -r '.required_topology.units[]' release.json); do
cp "dist/$cmd" "$name/"
done
cp deployments/INSTRUCTIONS.md CHANGELOG.md "$name/"
cp scripts/archie-update-* "$name/"
digest='${{ needs.build-and-push.outputs.agent_digest }}'
test -n "$digest"
jq --arg image "ghcr.io/samcharles93/archie-agent@$digest" '. + {agent_image: $image}' release.json > "$name/release.json"
# README.md doesn't exist yet in this repo; include it once it does
# rather than fail the release on its absence.
[ -f README.md ] && cp README.md "$name/"
zip -r "$name.zip" "$name"
sha256sum "$name.zip" > SHA256SUMS
echo "name=$name" >> "$GITHUB_OUTPUT"
- name: Upload distribution zip
uses: actions/upload-artifact@v4
with:
name: ${{ steps.zip.outputs.name }}
path: ${{ steps.zip.outputs.name }}.zip
path: |
${{ steps.zip.outputs.name }}.zip
SHA256SUMS
retention-days: 90
# A Release is what an operator can actually download from; the
@@ -370,7 +382,10 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -e
tag="archied/v${{ steps.version.outputs.version }}"
tag="v${{ steps.version.outputs.version }}"
if ! git ls-remote --exit-code --tags origin "refs/tags/$tag" >/dev/null; then
tag="archied/v${{ steps.version.outputs.version }}"
fi
version="${{ steps.version.outputs.version }}"
# The release body is this version's changelog section, not the whole
# file: from its "## [x.y.z]" heading to the next "## [" heading.
@@ -391,11 +406,11 @@ jobs:
# alone, so a re-run of this job (a retried or superseded deploy)
# cannot fail on an existing Release or duplicate the zip.
if gh release view "$tag" >/dev/null 2>&1; then
gh release upload "$tag" "${{ steps.zip.outputs.name }}.zip" --clobber
gh release upload "$tag" "${{ steps.zip.outputs.name }}.zip" SHA256SUMS --clobber
else
gh release create "$tag" \
--title "archied v$version" \
--notes-file "$notes" \
"${{ steps.zip.outputs.name }}.zip"
"${{ steps.zip.outputs.name }}.zip" SHA256SUMS
fi
echo "published $tag with ${{ steps.zip.outputs.name }}.zip"
+9
View File
@@ -16,6 +16,15 @@ func main() {
if archied.IsSetupArgs(args) {
os.Exit(archied.RunSetup(args[1:], os.Stdin, os.Stdout, os.Stderr))
}
if len(args) > 0 && args[0] == "select-release" {
os.Exit(archied.RunReleaseChannel(args[1:], os.Stdin, os.Stdout, os.Stderr))
}
if len(args) > 0 && args[0] == "update-image" {
os.Exit(archied.RunUpdateImage(args[1:], os.Stderr))
}
if len(args) > 0 && args[0] == "update-topology" {
os.Exit(archied.RunUpdateTopology(args[1:], os.Stdout, os.Stderr))
}
if len(args) > 0 && args[0] == "status" {
os.Exit(archied.RunStatus(args[1:], os.Stdout, os.Stderr))
}
+42
View File
@@ -86,3 +86,45 @@ only `archied` and `archie-agent` are independently versioned).
carries the Telegram, email and webhook channels since the v1.30.0 extraction, so
an install (or an upgrade across v1.30.0) needs its unit as well -- see
`deployments/systemd-user-service.md`.
### Native self-update
`archie-update-install` downloads the approved release's Linux/amd64 archive
and verifies its `SHA256SUMS` entry before executing or installing it. Releases
must include `release.json`, all host binaries, and the three updater scripts.
Older releases without this contract cannot be installed through the artifact
path. `--from-source` (or `ARCHIE_UPDATE_SOURCE_BUILD=1`) explicitly clones and
builds the selected release instead.
The update check command accepts `--channel stable` (default), `--channel next`
(includes prereleases), or `--channel exact-pin --pin 1.2.3`. Set these arguments
in the control plane's channel settings alongside the existing check command.
The notification destination remains separate from the release channel.
For scheduled, unattended runs, `archie-update-install --auto` discovers the
selected release and refuses if the daemon reports work in flight. It checks
again immediately before promotion. A scheduler can set
`ARCHIE_UPDATE_RELEASE_CHANNEL` and `ARCHIE_UPDATE_PIN`; nothing schedules updates
by default. All services stop through their ordinary graceful shutdown path.
`release.json` supplies the service order and required config sections to both
first-install and self-update. A missing unit or config section produces an
ordered migration plan and refuses installation. Apply that plan before retrying.
Automatic topology edits are intentionally unsupported.
The watchdog switches staged executables by rename, restarts services, and
requires the candidate daemon's existing readiness registry to remain healthy
for 120 seconds. Agent images come from the release manifest by digest. The
image switch updates the control-plane policy, preserving other settings;
rollback restores the prior image policy with conflict detection. Source builds
use the built image's immutable image ID.
Transactions live in `${XDG_STATE_HOME:-~/.local/state}/archie/update`, outside
the binary directory. `phase`, `environment`, `image.json` (when applicable),
and `previous/` describe the transaction. A lock reports the owning process.
On daemon boot, an interrupted transaction schedules independent recovery before
normal startup. To recover manually, run `transaction/watchdog --recover` from
that state directory. A failed recovery keeps the journal for another attempt.
Database snapshots remain available for manual recovery. Schema migrations and
PostgreSQL data are never automatically reversed.
+2 -1
View File
@@ -84,7 +84,6 @@ require (
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
golang.org/x/time v0.16.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 // indirect
@@ -119,8 +118,10 @@ require (
go.lsp.dev/jsonrpc2 v1.0.1
go.lsp.dev/protocol v1.0.1
go.lsp.dev/uri v1.0.1
golang.org/x/mod v0.41.0
golang.org/x/net v0.59.0
golang.org/x/oauth2 v0.37.0
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0
google.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e
google.golang.org/protobuf v1.36.12
+2
View File
@@ -270,6 +270,8 @@ golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU=
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
+8 -5
View File
@@ -112,7 +112,7 @@ if [ "$(uname -s)" != "Linux" ]; then
exit 1
fi
for cmd in git go; do
for cmd in git go jq; do
if ! command -v "$cmd" &>/dev/null; then
echo "Error: '$cmd' is required but not installed." >&2
exit 1
@@ -159,6 +159,8 @@ else
fi
fi
TOPOLOGY_UNITS="$(jq -er '.required_topology.units | join(" ")' "$SRC_DIR/release.json")"
# 4. Build and install the native daemon. archie-agent is deployed only as
# the managed task image; installing a host binary would imply an unsupported
# host execution path.
@@ -193,10 +195,10 @@ echo "==> Building native archie binaries..."
# webhook channels dead with no error anywhere. This list,
# the zip's two lists and the two in scripts/archie-update-install must agree;
# TestDistZipShipsEveryHostCommand fails when they do not.
for cmd in archied archie-gateway archie-state-store archie-ui archie-messaging; do
for cmd in $TOPOLOGY_UNITS; do
go build -ldflags "${LDFLAGS}" -o "${ARCHIE_BIN_DIR}/${cmd}" "./cmd/${cmd}"
done
install -m755 "${SRC_DIR}/scripts/archie-update-install" "${ARCHIE_BIN_DIR}/archie-update-install"
install -m755 "${SRC_DIR}"/scripts/archie-update-* "${ARCHIE_BIN_DIR}/"
)
echo " Installed archied, archie-gateway, archie-state-store, archie-ui, archie-messaging and updater to ${ARCHIE_BIN_DIR}/"
@@ -520,7 +522,8 @@ EOF
systemctl --user daemon-reload || true
SERVICE_INSTALLED=true
for unit in archied archie-state-store archie-gateway archie-ui archie-messaging; do
for unit in $TOPOLOGY_UNITS; do
test -f "${SYSTEMD_USER_DIR}/${unit}.service" || { echo "missing required unit: $unit" >&2; exit 1; }
echo " Installed ${SYSTEMD_USER_DIR}/${unit}.service"
done
@@ -546,7 +549,7 @@ EOF
# not race a dial it depends on.
if [ "${AUTO_START}" = true ]; then
echo "==> Enabling and starting Archie services..."
systemctl --user enable --now archie-state-store archie-gateway archied archie-ui archie-messaging ||
systemctl --user enable --now $TOPOLOGY_UNITS ||
echo " Notice: Could not start every service automatically. Check: systemctl --user status archied archie-state-store archie-gateway archie-ui archie-messaging"
fi
fi
+20 -2
View File
@@ -7,6 +7,7 @@ import (
"sync/atomic"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/buildinfo"
"github.com/samcharles93/archie-core/internal/domain/health"
)
@@ -24,6 +25,7 @@ import (
type healthSurface struct {
serving atomic.Bool
registry func() *health.Registry
inFlight func() int
}
// markServing tolerates a nil surface so a boot path that serves no health
@@ -70,7 +72,15 @@ func (h *healthSurface) handleDetailed(w http.ResponseWriter, r *http.Request) {
if report.Status != health.StatusOK {
w.WriteHeader(http.StatusServiceUnavailable)
}
_ = json.NewEncoder(w).Encode(report)
inFlight := -1
if h.serving.Load() && h.inFlight != nil {
inFlight = h.inFlight()
}
_ = json.NewEncoder(w).Encode(struct {
health.Report
Version string `json:"version"`
InFlight int `json:"in_flight"`
}{Report: report, Version: buildinfo.Version, InFlight: inFlight})
}
// startHealth binds the daemon's liveness surface on [health].listen. A
@@ -78,7 +88,15 @@ func (h *healthSurface) handleDetailed(w http.ResponseWriter, r *http.Request) {
// updates verified, and the watchdog would roll back every release it
// installs.
func (b *boot) startHealth(ctx context.Context) error {
b.health = &healthSurface{registry: func() *health.Registry { return b.healthRegistry }}
b.health = &healthSurface{
registry: func() *health.Registry { return b.healthRegistry },
inFlight: func() int {
if b.d == nil {
return -1
}
return b.d.InFlight()
},
}
addr := b.cfg.Health.Listen
if err := b.serveHealth(ctx, addr, b.health.handler(), "daemon health"); err != nil {
b.log.Error("health listener failed", "addr", addr, "err", err)
+7
View File
@@ -123,6 +123,13 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
return 0
}
if recovering, err := resumeUpdate(context.Background()); err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
} else if recovering {
return 0
}
root, cancelRoot := context.WithCancel(context.Background())
ctx, stop := signal.NotifyContext(root, os.Interrupt, syscall.SIGTERM)
defer stop()
+32
View File
@@ -0,0 +1,32 @@
package archied
import (
"flag"
"fmt"
"io"
"strings"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
func RunReleaseChannel(args []string, in io.Reader, out, stderr io.Writer) int {
flags := flag.NewFlagSet("select-release", flag.ContinueOnError)
flags.SetOutput(stderr)
channel := flags.String("channel", "stable", "stable, next, or exact-pin")
pin := flags.String("pin", "", "exact release version")
if err := flags.Parse(args); err != nil {
return 1
}
data, err := io.ReadAll(in)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
version, err := releaseupdate.SelectRelease(strings.Fields(string(data)), *channel, *pin)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
fmt.Fprintln(out, version)
return 0
}
+150
View File
@@ -0,0 +1,150 @@
package archied
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/BurntSushi/toml"
"golang.org/x/sys/unix"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
// RunUpdateTopology observes the host using the candidate's topology contract.
func RunUpdateTopology(args []string, out, stderr io.Writer) int {
flags := flag.NewFlagSet("update-topology", flag.ContinueOnError)
flags.SetOutput(stderr)
configPath := flags.String("config", "", "bootstrap config file")
releasePath := flags.String("release", "", "release manifest")
if err := flags.Parse(args); err != nil {
return 1
}
if err := checkUpdateTopology(*configPath, *releasePath, out); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
return 0
}
func checkUpdateTopology(configPath, releasePath string, out io.Writer) error {
data, err := os.ReadFile(releasePath)
if err != nil {
return err
}
var manifest struct {
Required releaseupdate.Topology `json:"required_topology"`
}
if err := json.Unmarshal(data, &manifest); err != nil {
return err
}
if len(manifest.Required.Units) == 0 {
return errors.New("release has no required_topology")
}
for _, unit := range manifest.Required.Units {
if !regexp.MustCompile(`^archie[a-z0-9-]*$`).MatchString(unit) {
return fmt.Errorf("invalid release unit %q", unit)
}
}
data, err = os.ReadFile(configPath)
if err != nil {
return err
}
var document map[string]any
if err := toml.Unmarshal(data, &document); err != nil {
return fmt.Errorf("observe topology: %w", err)
}
sections := make(map[string]bool)
for _, section := range manifest.Required.ConfigSections {
sections[section] = hasConfigSection(document, section)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
output, observeErr := exec.CommandContext(ctx, "systemctl", "--user", "list-unit-files", "archie*.service", "--no-legend", "--no-pager").Output()
units := observedUnits(output, observeErr)
plan := manifest.Required.MigrationPlan(units, sections)
if len(plan) != 0 {
fmt.Fprintln(out, "Required migration, in order:")
for _, step := range plan {
fmt.Fprintln(out, " -", step)
}
if observeErr != nil {
return fmt.Errorf("topology unknown; apply the printed migration before retrying: %w", observeErr)
}
return errors.New("refusing partial installation; apply the printed migration before retrying")
}
return nil
}
func hasConfigSection(document map[string]any, section string) bool {
for part := range strings.SplitSeq(section, ".") {
next, ok := document[part].(map[string]any)
if !ok {
return false
}
document = next
}
return true
}
func updateStateDir() string {
if dir := os.Getenv("ARCHIE_UPDATE_STATE_DIR"); dir != "" {
return dir
}
dir := os.Getenv("XDG_STATE_HOME")
if dir == "" {
home, _ := os.UserHomeDir()
dir = filepath.Join(home, ".local", "state")
}
return filepath.Join(dir, "archie", "update")
}
// Recovery runs independently of the daemon unit: it must be able to stop that
// unit and replace its executable without systemd killing the recovery process.
func resumeUpdate(ctx context.Context) (bool, error) {
journal := filepath.Join(updateStateDir(), "transaction")
if _, err := os.Stat(journal); errors.Is(err, os.ErrNotExist) {
return false, nil
} else if err != nil {
return false, err
}
lock, err := os.OpenFile(filepath.Join(updateStateDir(), "lock"), os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return false, err
}
defer lock.Close()
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB); errors.Is(err, unix.EWOULDBLOCK) {
return false, nil
} else if err != nil {
return false, err
}
defer func() { _ = unix.Flock(int(lock.Fd()), unix.LOCK_UN) }()
cmd := exec.CommandContext(ctx, "systemd-run", "--user", "--collect", "--quiet", "--unit=archied-update-recovery", "--setenv=ARCHIE_UPDATE_STATE_DIR="+updateStateDir(), filepath.Join(journal, "watchdog"), "--boot-recover")
if output, err := cmd.CombinedOutput(); err != nil {
return false, fmt.Errorf("schedule update recovery: %w: %s", err, output)
}
return true, nil
}
func observedUnits(output []byte, err error) map[string]bool {
units := make(map[string]bool)
if err != nil {
return units
}
for line := range strings.SplitSeq(string(output), "\n") {
fields := strings.Fields(line)
if len(fields) >= 2 && fields[1] != "masked" {
units[strings.TrimSuffix(fields[0], ".service")] = true
}
}
return units
}
+117
View File
@@ -0,0 +1,117 @@
package archied
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log/slog"
"os"
"path/filepath"
"regexp"
"time"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/app/servicekit"
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/secret"
)
type imageChange struct {
Previous string `json:"previous"`
Candidate string `json:"candidate"`
}
// RunUpdateImage journals and switches the control-plane image policy. It only
// changes the image field, preserving concurrent edits to unrelated settings.
func RunUpdateImage(args []string, stderr io.Writer) int {
flags := flag.NewFlagSet("update-image", flag.ContinueOnError)
flags.SetOutput(stderr)
configPath := flags.String("config", "", "bootstrap config")
journal := flags.String("journal", "", "transaction directory")
candidate := flags.String("prepare", "", "candidate image digest")
rollback := flags.Bool("rollback", false, "restore the prior image")
if err := flags.Parse(args); err != nil {
return 1
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := updateImage(ctx, *configPath, *journal, *candidate, *rollback); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
return 0
}
func updateImage(ctx context.Context, configPath, journal, candidate string, rollback bool) error {
_, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), configPath, "")
if err != nil {
return err
}
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, doc.Config.Services, secret.NewRegistry())
if err != nil {
return err
}
defer cleanup()
rpc := client.ControlPlane()
response, err := rpc.Query(ctx, &pb.QueryRequest{Kind: controlplane.ContainerRuntimePoliciesKind})
if err != nil {
return err
}
var policy map[string]json.RawMessage
if err := json.Unmarshal(response.Resource.ValueJson, &policy); err != nil {
return err
}
var current string
if err := json.Unmarshal(policy["image"], &current); err != nil {
return err
}
path := filepath.Join(journal, "image.json")
if candidate != "" {
return prepareImageChange(path, current, candidate)
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
var change imageChange
if err := json.Unmarshal(data, &change); err != nil {
return err
}
expected, desired := change.Previous, change.Candidate
if rollback {
expected, desired = desired, expected
}
if current == desired {
return nil
}
if current != expected {
return errors.New("image policy changed during update; refusing to overwrite it")
}
policy["image"], err = json.Marshal(desired)
if err != nil {
return err
}
data, err = json.Marshal(policy)
if err != nil {
return err
}
_, err = rpc.Command(ctx, &pb.CommandRequest{Kind: controlplane.ContainerRuntimePoliciesKind, Command: "replace", ValueJson: data, ExpectedVersion: response.Resource.Version, Actor: "self-update", Source: "installer", RequestId: fmt.Sprintf("self-update-%x-%d", sha256.Sum256(data), response.Resource.Version)})
return err
}
func prepareImageChange(path, current, candidate string) error {
if !regexp.MustCompile(`^([a-zA-Z0-9./:_-]+@)?sha256:[a-f0-9]{64}$`).MatchString(candidate) {
return errors.New("candidate image must be pinned by digest")
}
data, err := json.Marshal(imageChange{Previous: current, Candidate: candidate})
if err != nil {
return err
}
return os.WriteFile(path, data, 0o600)
}
+75
View File
@@ -0,0 +1,75 @@
package archied
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"golang.org/x/sys/unix"
)
func TestUpdateTopologyRefusal(t *testing.T) {
root := t.TempDir()
manifest := filepath.Join(root, "release.json")
config := filepath.Join(root, "config.toml")
writeUpdateFixture(t, manifest, `{"required_topology":{"units":["archie-state-store","archie-gateway","archied"],"config_sections":["services.state","services.gateway"]}}`)
writeUpdateFixture(t, config, "[services.state]\ntarget='localhost:9090'\n")
writeUpdateFixture(t, filepath.Join(root, "systemctl"), "#!/bin/sh\nprintf 'archied.service enabled enabled\\n'\n")
t.Setenv("PATH", root+":"+os.Getenv("PATH"))
var output bytes.Buffer
if err := checkUpdateTopology(config, manifest, &output); err == nil {
t.Fatal("partial topology accepted")
}
for _, missing := range []string{"configure [services.gateway]", "install and enable archie-state-store.service", "install and enable archie-gateway.service"} {
if !strings.Contains(output.String(), missing) {
t.Fatalf("missing step %q: %s", missing, output.String())
}
}
}
func TestResumeUpdate(t *testing.T) {
for _, active := range []bool{false, true} {
t.Run(map[bool]string{false: "interrupted", true: "live"}[active], func(t *testing.T) {
root := t.TempDir()
t.Setenv("ARCHIE_UPDATE_STATE_DIR", root)
if err := os.Mkdir(filepath.Join(root, "transaction"), 0o700); err != nil {
t.Fatal(err)
}
called := filepath.Join(root, "called")
writeUpdateFixture(t, filepath.Join(root, "systemd-run"), "#!/bin/sh\nprintf '%s\\n' \"$@\" > '"+called+"'\n")
t.Setenv("PATH", root+":"+os.Getenv("PATH"))
lock, err := os.OpenFile(filepath.Join(root, "lock"), os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
t.Fatal(err)
}
defer lock.Close()
if active {
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil {
t.Fatal(err)
}
}
resumed, err := resumeUpdate(t.Context())
if err != nil || resumed == active {
t.Fatalf("resumed=%v err=%v", resumed, err)
}
if !active {
data, err := os.ReadFile(called)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(data), "--boot-recover") {
t.Fatalf("not a rollback: %s", data)
}
}
})
}
}
func writeUpdateFixture(t *testing.T, path, data string) {
t.Helper()
if err := os.WriteFile(path, []byte(data), 0o700); err != nil {
t.Fatal(err)
}
}
+26 -4
View File
@@ -944,7 +944,7 @@ type taskDispatcher struct {
active int
slotsFree *sync.Cond
repoTail map[string]chan struct{}
wg sync.WaitGroup
pending int
}
// SetMaxConcurrency accepts a new global limit without a restart. Running
@@ -1004,7 +1004,16 @@ func (d *taskDispatcher) Submit(
d.mu.Unlock()
}
d.wg.Go(func() {
d.mu.Lock()
d.pending++
d.mu.Unlock()
go func() {
defer func() {
d.mu.Lock()
d.pending--
d.mu.Unlock()
d.slotsFree.Broadcast()
}()
if previous != nil {
<-previous
}
@@ -1021,11 +1030,15 @@ func (d *taskDispatcher) Submit(
}()
}
process(ctx, task)
})
}()
}
func (d *taskDispatcher) Wait() {
d.wg.Wait()
d.mu.Lock()
defer d.mu.Unlock()
for d.pending > 0 {
d.slotsFree.Wait()
}
}
func (d *Daemon) poll(ctx context.Context) {
@@ -2214,3 +2227,12 @@ func (d *Daemon) RemoveWorktree(owner, repo, identity string, issue int) error {
}
return trees.Cleanup(owner, repo, issue)
}
// InFlight counts the same submitted work that graceful shutdown waits for,
// including tasks waiting for a repository or concurrency slot.
func (d *Daemon) InFlight() int {
dispatcher := d.taskDispatcher()
dispatcher.mu.Lock()
defer dispatcher.mu.Unlock()
return dispatcher.pending
}
+39
View File
@@ -0,0 +1,39 @@
package daemon
import (
"context"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/domain/workflow"
)
func TestDispatcherWaitIncludesQueuedWork(t *testing.T) {
dispatcher := newTaskDispatcher(1, nil)
started := make(chan struct{}, 2)
release := make(chan struct{})
process := func() { started <- struct{}{}; <-release }
for range 2 {
dispatcher.Submit(t.Context(), &workflow.Task{Owner: "o", Repo: "r"}, func(_ context.Context, _ *workflow.Task) { process() })
}
<-started
dispatcher.mu.Lock()
count := dispatcher.pending
dispatcher.mu.Unlock()
if count != 2 {
t.Fatalf("in-flight count=%d, want running and queued task", count)
}
done := make(chan struct{})
go func() { dispatcher.Wait(); close(done) }()
select {
case <-done:
t.Fatal("drain completed with pending work")
default:
}
close(release)
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("drain did not finish")
}
}
+54
View File
@@ -0,0 +1,54 @@
package releaseupdate
import (
"fmt"
"regexp"
"strings"
"golang.org/x/mod/semver"
)
// SelectRelease compares release versions, never git-describe strings. Exact
// pins still have to exist in the catalog; a pin cannot invent a release.
func SelectRelease(tags []string, channel, pin string) (string, error) {
if channel == "" {
channel = "stable"
}
if channel != "stable" && channel != "next" && channel != "exact-pin" {
return "", fmt.Errorf("unknown release channel %q", channel)
}
pin = strings.TrimPrefix(pin, "v")
if channel == "exact-pin" && !semver.IsValid("v"+pin) {
return "", fmt.Errorf("exact-pin requires a semantic version")
}
fullVersion := regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+([+-].*)?$`)
newest := ""
for _, tag := range tags {
tag = strings.TrimPrefix(tag, "refs/tags/")
tag = strings.TrimPrefix(tag, "archied/")
if !fullVersion.MatchString(tag) || !semver.IsValid(tag) {
continue
}
if !releaseMatchesChannel(tag, channel, pin) {
continue
}
if semver.Compare(tag, newest) > 0 {
newest = tag
}
}
if newest == "" {
return "", fmt.Errorf("no release found for channel %s (pin %s)", channel, pin)
}
return strings.TrimPrefix(newest, "v"), nil
}
func releaseMatchesChannel(tag, channel, pin string) bool {
switch channel {
case "stable":
return semver.Prerelease(tag) == ""
case "exact-pin":
return tag == "v"+pin
default:
return true
}
}
+24
View File
@@ -0,0 +1,24 @@
package releaseupdate
import "testing"
func TestSelectRelease(t *testing.T) {
tags := []string{"archied/v1.9.0", "v1.10.0", "archied/v2.0.0-rc.2", "v2.0.0-rc.10", "archie/v9.0.0", "nonsense"}
for _, tc := range []struct{ channel, pin, want string }{
{"stable", "", "1.10.0"},
{"", "", "1.10.0"},
{"next", "", "2.0.0-rc.10"},
{"exact-pin", "1.9.0", "1.9.0"},
{"exact-pin", "2.0.0-rc.2", "2.0.0-rc.2"},
{"exact-pin", "9.0.0", ""},
{"exact-pin", "", ""},
{"invalid", "", ""},
} {
t.Run(tc.channel+tc.pin, func(t *testing.T) {
got, err := SelectRelease(tags, tc.channel, tc.pin)
if got != tc.want || (err != nil) != (tc.want == "") {
t.Fatalf("got %q, %v; want %q", got, err, tc.want)
}
})
}
}
+10 -2
View File
@@ -19,13 +19,21 @@ const updateResultSentinel = "ARCHIE_UPDATE_RESULT "
// CommandCatalog reads a Snapshot JSON document from an explicitly configured
// argv command. It never invokes a shell; deployment tooling remains an
// administrator-owned adapter.
type CommandCatalog struct{ Command []string }
type CommandCatalog struct {
Command []string
ReleaseChannel string
Pin string
}
func (c CommandCatalog) Check(ctx context.Context) (Snapshot, error) {
if len(c.Command) == 0 {
return Snapshot{}, fmt.Errorf("update check command is empty")
}
output, err := exec.CommandContext(ctx, c.Command[0], c.Command[1:]...).Output()
cmd := exec.CommandContext(ctx, c.Command[0], c.Command[1:]...)
if c.ReleaseChannel != "" {
cmd.Env = append(cmd.Environ(), "ARCHIE_UPDATE_RELEASE_CHANNEL="+c.ReleaseChannel, "ARCHIE_UPDATE_PIN="+c.Pin)
}
output, err := cmd.Output()
if err != nil {
return Snapshot{}, fmt.Errorf("run update check: %w", err)
}
+215
View File
@@ -0,0 +1,215 @@
package releaseupdate
import (
"archive/zip"
"crypto/sha256"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"golang.org/x/sys/unix"
)
func TestUpdateTransaction(t *testing.T) {
for _, scenario := range []string{"healthy", "checksum", "unsafe-path", "missing-digest", "busy", "lock", "unhealthy", "crash"} {
t.Run(scenario, func(t *testing.T) {
f := newUpdateFixture(t, scenario)
var lock *os.File
if scenario == "lock" {
var err error
lock, err = os.OpenFile(filepath.Join(f.state, "lock"), os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
t.Fatal(err)
}
defer lock.Close()
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil {
t.Fatal(err)
}
fixtureWrite(t, filepath.Join(f.state, "holder"), "fixture-owner", 0o600)
}
args := []string{}
if scenario == "busy" {
args = append(args, "--auto")
}
output, err := f.run(t, f.installer, args...)
refusal := map[string]string{"checksum": "checksum mismatch", "unsafe-path": "unsafe archive path", "missing-digest": "digest-pinned", "busy": "work in flight", "lock": "fixture-owner"}[scenario]
if refusal != "" {
if err == nil || !strings.Contains(output, refusal) {
t.Fatalf("wanted %q refusal; err=%v output=%s", refusal, err, output)
}
f.assertVersion(t, "old")
return
}
if err != nil {
t.Fatalf("installer: %v\n%s", err, output)
}
f.assertVersion(t, "old") // Staging cannot mutate a running executable.
old, err := os.Open(filepath.Join(f.bin, "archied"))
if err != nil {
t.Fatal(err)
}
defer old.Close()
watchdog := filepath.Join(f.state, "transaction", "watchdog")
output, err = f.run(t, watchdog)
if scenario == "crash" {
if err == nil {
t.Fatal("expected injected crash")
}
f.env = append(f.env, "CRASH_AFTER=")
output, err = f.run(t, watchdog, "--boot-recover")
}
if err != nil {
t.Fatalf("watchdog: %v\n%s", err, output)
}
want := "old"
if scenario == "healthy" {
want = "new"
}
f.assertVersion(t, want)
data := make([]byte, 100)
n, _ := old.Read(data)
if !strings.Contains(string(data[:n]), "old") {
t.Fatal("promotion overwrote the open old inode")
}
if _, err := os.Stat(filepath.Join(f.state, "transaction")); !os.IsNotExist(err) {
t.Fatalf("journal not retired: %v", err)
}
})
}
}
type updateFixture struct {
bin, state, installer string
env []string
}
func fixtureWrite(t *testing.T, path, data string, mode os.FileMode) {
t.Helper()
if err := os.WriteFile(path, []byte(data), mode); err != nil {
t.Fatal(err)
}
}
func newUpdateFixture(t *testing.T, scenario string) updateFixture {
t.Helper()
root := t.TempDir()
bin, state, fake := filepath.Join(root, "bin"), filepath.Join(root, "state"), filepath.Join(root, "fake")
for _, dir := range []string{bin, state, fake} {
if err := os.MkdirAll(dir, 0o700); err != nil {
t.Fatal(err)
}
}
scripts, err := filepath.Abs("../../scripts")
if err != nil {
t.Fatal(err)
}
for _, name := range []string{"archied", "archie-state-store", "archie-gateway", "archie-ui", "archie-messaging"} {
fixtureWrite(t, filepath.Join(bin, name), "#!/bin/bash\n# old\nexit 0\n", 0o755)
}
config := filepath.Join(root, "config.toml")
fixtureWrite(t, config, "[services.state]\n[services.gateway]\n", 0o600)
archive := makeUpdateArchive(t, root, scripts, scenario)
sum := sha256.Sum256(archive)
if scenario == "checksum" {
sum[0]++
}
fixtureWrite(t, filepath.Join(root, "SHA256SUMS"), fmt.Sprintf("%x archie-core-archied-v2.0.0-linux-amd64.zip\n", sum), 0o600)
fixtureWrite(t, filepath.Join(fake, "curl"), `#!/bin/bash
out=""
for ((i=1;i<=$#;i++)); do
if [ "${!i}" = -o ]; then j=$((i+1)); out="${!j}"; fi
case "${!i}" in
*/SHA256SUMS) source="$FIXTURE/SHA256SUMS" ;;
*.zip) source="$FIXTURE/archive.zip" ;;
esac
done
if [ -n "$out" ]; then cp "$source" "$out"; exit; fi
if [ "$SCENARIO" = unhealthy ]; then exit 22; fi
busy=0
[ "$SCENARIO" = busy ] && busy=1
printf '{"status":"ok","version":"2.0.0","in_flight":%s}\n' "$busy"
`, 0o755)
for _, name := range []string{"systemctl", "systemd-run", "docker"} {
fixtureWrite(t, filepath.Join(fake, name), "#!/bin/bash\nexit 0\n", 0o755)
}
fixtureWrite(t, filepath.Join(fake, "git"), "#!/bin/bash\necho unexpected-git >&2\nexit 98\n", 0o755)
fixtureWrite(t, filepath.Join(fake, "mv"), `#!/bin/bash
/bin/mv "$@" || exit
if [ -n "$CRASH_AFTER" ] && [[ "$*" == *".archied.new"* ]]; then kill -KILL "$PPID"; fi
`, 0o755)
return updateFixture{bin: bin, state: state, installer: filepath.Join(scripts, "archie-update-install"), env: append(os.Environ(),
"PATH="+fake+":"+os.Getenv("PATH"), "FIXTURE="+root, "SCENARIO="+scenario,
"ARCHIE_BIN_DIR="+bin, "ARCHIE_CONFIG_PATH="+config, "ARCHIE_UPDATE_STATE_DIR="+state,
"ARCHIE_UPDATE_DAEMON_PREVIOUS=1.0.0", "ARCHIE_UPDATE_DAEMON_VERSION=2.0.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS=1.0.0", "ARCHIE_UPDATE_AGENT_VERSION="+map[bool]string{true: "2.0.0"}[scenario == "missing-digest"],
"ARCHIE_UPDATE_HEALTH_TIMEOUT=1", "ARCHIE_UPDATE_OBSERVE_SECONDS=0", "ARCHIE_UPDATE_POLL_INTERVAL=0.01",
"ARCHIE_UPDATE_STATE_STORE_TARGET=", "CRASH_AFTER="+map[bool]string{true: "archied"}[scenario == "crash"],
)}
}
func makeUpdateArchive(t *testing.T, root, scripts, scenario string) []byte {
t.Helper()
file, err := os.Create(filepath.Join(root, "archive.zip"))
if err != nil {
t.Fatal(err)
}
z := zip.NewWriter(file)
files := map[string]string{"release.json": `{"required_topology":{"units":["archie-state-store","archie-gateway","archie-ui","archie-messaging","archied"],"config_sections":["services.state","services.gateway"]},"agent_image":"mutable:latest"}`}
for _, name := range []string{"archied", "archie-state-store", "archie-gateway", "archie-ui", "archie-messaging"} {
files[name] = "#!/bin/bash\n# new\nif [ \"$1\" = -version ]; then echo '" + name + " 2.0.0'; fi\nexit 0\n"
}
for _, name := range []string{"archie-update-install", "archie-update-watchdog", "archie-update-check"} {
data, err := os.ReadFile(filepath.Join(scripts, name))
if err != nil {
t.Fatal(err)
}
files[name] = string(data)
}
if scenario == "unsafe-path" {
files["../escape"] = "escape"
}
for name, body := range files {
header := &zip.FileHeader{Name: "archie-core-archied-v2.0.0-linux-amd64/" + name, Method: zip.Deflate}
header.SetMode(0o755)
w, err := z.CreateHeader(header)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
if err := z.Close(); err != nil {
t.Fatal(err)
}
if err := file.Close(); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(file.Name())
if err != nil {
t.Fatal(err)
}
return data
}
func (f updateFixture) run(t *testing.T, path string, args ...string) (string, error) {
t.Helper()
cmd := exec.CommandContext(t.Context(), path, args...)
cmd.Env = f.env
data, err := cmd.CombinedOutput()
return string(data), err
}
func (f updateFixture) assertVersion(t *testing.T, want string) {
t.Helper()
data, err := os.ReadFile(filepath.Join(f.bin, "archied"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(data), "# "+want) {
t.Fatalf("wanted %s executable: %s", want, data)
}
}
+25
View File
@@ -0,0 +1,25 @@
package releaseupdate
import "fmt"
type Topology struct {
Units []string `json:"units"`
ConfigSections []string `json:"config_sections"`
}
// MigrationPlan preserves the release's dependency order. Unknown observations
// require every step to be checked manually rather than assuming a partial host.
func (t Topology) MigrationPlan(units, sections map[string]bool) []string {
var steps []string
for _, section := range t.ConfigSections {
if !sections[section] {
steps = append(steps, fmt.Sprintf("configure [%s]", section))
}
}
for _, unit := range t.Units {
if !units[unit] {
steps = append(steps, fmt.Sprintf("install and enable %s.service", unit))
}
}
return steps
}
+25
View File
@@ -0,0 +1,25 @@
package releaseupdate
import (
"reflect"
"testing"
)
func TestMigrationPlan(t *testing.T) {
topology := Topology{Units: []string{"state", "gateway", "ui", "daemon"}, ConfigSections: []string{"services.state"}}
for _, tc := range []struct {
name string
units, sections map[string]bool
want []string
}{
{"unknown", nil, nil, []string{"configure [services.state]", "install and enable state.service", "install and enable gateway.service", "install and enable ui.service", "install and enable daemon.service"}},
{"missing", map[string]bool{"state": true, "daemon": true}, map[string]bool{"services.state": true}, []string{"install and enable gateway.service", "install and enable ui.service"}},
{"complete", map[string]bool{"state": true, "gateway": true, "ui": true, "daemon": true}, map[string]bool{"services.state": true}, nil},
} {
t.Run(tc.name, func(t *testing.T) {
if got := topology.MigrationPlan(tc.units, tc.sections); !reflect.DeepEqual(got, tc.want) {
t.Fatalf("got %v, want %v", got, tc.want)
}
})
}
}
+6
View File
@@ -0,0 +1,6 @@
{
"required_topology": {
"units": ["archie-state-store", "archie-gateway", "archie-ui", "archie-messaging", "archied"],
"config_sections": ["services.state", "services.gateway"]
}
}
+25 -27
View File
@@ -46,7 +46,7 @@ readonly CURL_TIMEOUT="${ARCHIE_FORGE_TIMEOUT:-20}"
readonly TOKEN="${ARCHIE_FORGE_TOKEN:-}"
# base 1.7.0-4-g0736708 -> 1.7.0
base() { printf '%s\n' "${1%%-*}"; }
base() { printf '%s\n' "$1" | sed -E 's/-[0-9]+-g[0-9a-f]+$//'; }
# installed_for reads the version this binary reports for a named component.
# archied stamps both its own version and the runtime version the release was
@@ -72,35 +72,33 @@ curl_api() {
fi
}
# newest_tag_git resolves the newest tag for a component prefix from the git
# remote. No local checkout, no working-tree state, no fetch -- `ls-remote`
# queries the remote directly, so this works on a host that has never cloned
# anything.
RELEASE_CHANNEL="${ARCHIE_UPDATE_RELEASE_CHANNEL:-stable}"
PIN="${ARCHIE_UPDATE_PIN:-}"
while [ "$#" -gt 0 ]; do
case "$1" in
--channel) RELEASE_CHANNEL="$2"; shift 2 ;;
--pin) PIN="$2"; shift 2 ;;
*) echo "unknown option: $1" >&2; exit 1 ;;
esac
done
select_release() { "$BIN" select-release -channel "$RELEASE_CHANNEL" -pin "$PIN"; }
newest_tag_git() {
local component="$1" tags
command -v git >/dev/null 2>&1 || return 1
tags="$(git ls-remote --tags --refs "$SOURCE_URL" "$component/v*" 2>/dev/null)" || return 1
[ -n "$tags" ] || return 1
printf '%s\n' "$tags" | sed 's|.*refs/tags/||' | sed "s|^$component/v||" | sort -V | tail -1
local tags
tags="$(git ls-remote --tags --refs "$SOURCE_URL" 'archied/v*' 'v*' 2>/dev/null)" || return 1
printf '%s\n' "$tags" | awk '{print $2}' | select_release
}
# newest_tag_api is the API equivalent, used when git is unavailable.
newest_tag_api() {
local component="$1" page payload found total
for page in 1 2 3; do
payload="$(curl_api "${API_BASE}/repos/${REPO_SLUG}/tags?per_page=100&page=${page}")" || return 1
[ -n "$payload" ] || return 1
found="$(printf '%s\n' "$payload" \
| grep -o "\"name\":[[:space:]]*\"${component}/v[^\"]*\"" \
| sed 's|.*"'"${component}"'/v||; s|"$||' | sort -V | tail -1)"
if [ -n "$found" ]; then
printf '%s\n' "$found"
return 0
fi
total="$(printf '%s\n' "$payload" | grep -c '"name"')"
[ "${total:-0}" -lt 100 ] && break
done
return 1
local page payload tags=""
for page in $(seq 1 100); do
payload="$(curl_api "${API_BASE}/repos/${REPO_SLUG}/tags?per_page=100&page=${page}")" || return 1
tags+="$(jq -er '.[].name' <<< "$payload")"$'\n'
if [ "$(jq 'length' <<< "$payload")" -lt 100 ]; then
printf '%s' "$tags" | select_release
return $?
fi
done
echo "tag catalog exceeded pagination limit" >&2
return 1
}
# artifact_state reports whether a tagged release is PUBLISHED with assets,
+135 -151
View File
@@ -16,34 +16,8 @@ BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}"
export PATH="/usr/local/go/bin:$PATH"
configured_agent_image() {
local image
image="$(awk '
/^\[containers\][[:space:]]*(#.*)?$/ { in_containers = 1; next }
/^\[/ { in_containers = 0 }
in_containers && /^[[:space:]]*image[[:space:]]*=/ {
value = $0
sub(/^[[:space:]]*image[[:space:]]*=[[:space:]]*/, "", value)
sub(/[[:space:]]*(#.*)?$/, "", value)
if (value ~ /^".*"$/) {
sub(/^"/, "", value)
sub(/"$/, "", value)
print value
} else if (length(value) >= 2 && substr(value, 1, 1) == sprintf("%c", 39) && substr(value, length(value), 1) == sprintf("%c", 39)) {
print substr(value, 2, length(value) - 2)
}
exit
}
' "$CONFIG_PATH" 2>/dev/null || true)"
printf '%s' "$image"
}
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-$(configured_agent_image)}"
AGENT_IMAGE="${AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
AGENT_IMAGE_BACKUP="$BIN_DIR/archie-agent-image.prev.tar"
# State for the managed image, independent of archied's daemon build stamp.
AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}"
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
# The address the daemon dials for its state, so the watchdog can wait for it to
# accept connections before it starts archied. Read with a narrow awk: the
@@ -67,9 +41,27 @@ configured_state_store_target() {
STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-$(configured_state_store_target)}"
GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui archie-messaging"
GATEWAY_SERVICES="archie-state-store archie-gateway archie-ui archie-messaging archied"
GATEWAY_BINARIES="$GATEWAY_SERVICES"
SOURCE_BUILD="${ARCHIE_UPDATE_SOURCE_BUILD:-0}"
AUTO=0
for arg in "$@"; do
case "$arg" in
--from-source) SOURCE_BUILD=1 ;;
--auto) AUTO=1 ;;
*) echo "unknown option: $arg" >&2; exit 1 ;;
esac
done
if [ "$AUTO" = 1 ] && [ -z "${ARCHIE_UPDATE_DAEMON_VERSION:-}${ARCHIE_UPDATE_AGENT_VERSION:-}" ]; then
snapshot="$("$(dirname "$0")/archie-update-check")"
export ARCHIE_UPDATE_DAEMON_PREVIOUS="$(jq -r '.Components[] | select(.ID == "daemon") | .Installed' <<< "$snapshot")"
export ARCHIE_UPDATE_DAEMON_VERSION="$(jq -r '.Components[] | select(.ID == "daemon") | .Available' <<< "$snapshot")"
export ARCHIE_UPDATE_AGENT_PREVIOUS="$(jq -r '.Components[] | select(.ID == "agent") | .Installed' <<< "$snapshot")"
export ARCHIE_UPDATE_AGENT_VERSION="$(jq -r '.Components[] | select(.ID == "agent") | .Available' <<< "$snapshot")"
fi
PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}"
PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}"
GW_TARGET="${ARCHIE_UPDATE_DAEMON_VERSION:-}"
@@ -132,146 +124,138 @@ for target in "$GW_TARGET" "$RT_TARGET"; do
fi
done
STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}"
mkdir -p "$STATE_DIR"
chmod 700 "$STATE_DIR"
exec 9>"$STATE_DIR/lock"
if ! flock -n 9; then
echo "update locked by $(cat "$STATE_DIR/holder" 2>/dev/null); journal: $STATE_DIR/transaction" >&2
exit 1
fi
printf 'installer PID %s\n' "$$" > "$STATE_DIR/holder"
if [ -d "$STATE_DIR/transaction" ]; then
echo "unfinished update: run $STATE_DIR/transaction/watchdog --recover before retrying" >&2
exit 1
fi
tmp="$(mktemp -d)"
source_dir="$tmp/source"
trap 'rm -rf "$tmp"' EXIT
echo "==> fetching approved component releases"
git clone --quiet --filter=blob:none --no-checkout "$SOURCE_URL" "$source_dir"
git -C "$source_dir" fetch --quiet --tags origin
gw_commit=""
rt_commit=""
if [[ ",$components," == *,daemon,* ]]; then
gw_commit="$(git -C "$source_dir" rev-parse --verify "refs/tags/archied/v$GW^{commit}")"
source_dir="$tmp/source"
release_dir="$tmp/release"
mkdir -p "$release_dir"
release_version="${GW_TARGET:-$RT_TARGET}"
if [ -n "$GW_TARGET" ] && [ -n "$RT_TARGET" ] && [ "$GW_TARGET" != "$RT_TARGET" ]; then
echo "daemon and agent must name the same release" >&2; exit 1
fi
if [ "$SOURCE_BUILD" = 1 ]; then
tag="v$release_version"
git ls-remote --exit-code --tags "$SOURCE_URL" "refs/tags/$tag" >/dev/null || tag="archied/v$release_version"
git clone --quiet --depth=1 --branch "$tag" "$SOURCE_URL" "$source_dir"
cp "$source_dir/release.json" "$release_dir/"
GATEWAY_BINARIES="$(jq -er '.required_topology.units | join(" ")' "$release_dir/release.json")"
cp "$source_dir"/scripts/archie-update-* "$release_dir/"
if [[ ",$components," == *,daemon,* ]]; then
for cmd in $GATEWAY_BINARIES; do
[[ "$cmd" =~ ^archie[a-z0-9-]*$ ]] || { echo "invalid release unit: $cmd" >&2; exit 1; }
(cd "$source_dir" && go build -ldflags "-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$GW -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$RT" -o "$release_dir/$cmd" "./cmd/$cmd")
done
fi
else
case "$(uname -s)/$(uname -m)" in
Linux/x86_64) platform=linux-amd64 ;;
*) echo "no published artifact for $(uname -s)/$(uname -m)" >&2; exit 1 ;;
esac
name="archie-core-archied-v$release_version-$platform"
base="${ARCHIE_RELEASE_BASE_URL:-https://github.com/samcharles93/archie-core/releases/download}/v$release_version"
if ! curl -fLSs --max-time 300 "$base/$name.zip" -o "$tmp/$name.zip"; then
base="${ARCHIE_RELEASE_BASE_URL:-https://github.com/samcharles93/archie-core/releases/download}/archied/v$release_version"
curl -fLSs --max-time 300 "$base/$name.zip" -o "$tmp/$name.zip"
fi
curl -fLSs --max-time 60 "$base/SHA256SUMS" -o "$tmp/SHA256SUMS"
expected="$(awk -v name="$name.zip" '$2 == name {print $1}' "$tmp/SHA256SUMS")"
[[ "$expected" =~ ^[a-fA-F0-9]{64}$ ]] || { echo "missing or ambiguous archive checksum" >&2; exit 1; }
printf '%s %s\n' "$expected" "$tmp/$name.zip" | sha256sum --check --status || { echo "release checksum mismatch" >&2; exit 1; }
# Reject paths outside the single distribution directory before extracting.
while IFS= read -r entry; do
case "$entry" in "$name/"*) ;; *) echo "unsafe archive path: $entry" >&2; exit 1 ;; esac
case "/$entry/" in */../*|*/./*|*\\*) echo "unsafe archive path: $entry" >&2; exit 1 ;; esac
done < <(unzip -Z1 "$tmp/$name.zip")
if unzip -Z -l "$tmp/$name.zip" | awk '$1 ~ /^l/ {found=1} END {exit !found}'; then
echo "release contains symlinks" >&2; exit 1
fi
unzip -q "$tmp/$name.zip" -d "$tmp/unpacked"
if find "$tmp/unpacked" -type l | grep -q .; then echo "release contains symlinks" >&2; exit 1; fi
cp -a "$tmp/unpacked/$name/." "$release_dir/"
fi
# Observation and config parsing belong to the daemon, including on old hosts.
observer="$release_dir/archied"
[ -x "$observer" ] || observer="$BIN_DIR/archied"
"$observer" update-topology -config "$CONFIG_PATH" -release "$release_dir/release.json"
GATEWAY_SERVICES="$(jq -er '.required_topology.units | join(" ")' "$release_dir/release.json")"
GATEWAY_BINARIES="$GATEWAY_SERVICES"
if [[ ",$components," == *,agent,* ]]; then
rt_commit="$(git -C "$source_dir" rev-parse --verify "refs/tags/archied/v$RT^{commit}")"
fi
build_commit="$gw_commit"
[ -n "$build_commit" ] || build_commit="$rt_commit"
git -C "$source_dir" checkout --quiet --detach "$build_commit"
cd "$source_dir"
if [[ ",$components," == *,daemon,* ]]; then
missing_units=""
for svc in $GATEWAY_SERVICES; do
[ -d "$source_dir/cmd/$svc" ] || continue
if ! systemctl --user list-unit-files "$svc.service" --no-legend 2>/dev/null | grep -q .; then
missing_units="${missing_units:+$missing_units }$svc.service"
fi
done
if [ -n "$missing_units" ]; then
echo "refusing to install archied $GW: it runs as separate processes and this host has no unit for: $missing_units" >&2
echo "create them from deployments/systemd-user-service.md, then retry the update" >&2
exit 1
fi
unmanaged_units=""
for dir in "$source_dir"/cmd/*/; do
[ -d "$dir" ] || continue
svc="$(basename "$dir")"
case " $GATEWAY_SERVICES " in
*" $svc "*) continue ;;
esac
if systemctl --user list-unit-files "$svc.service" --no-legend 2>/dev/null | grep -q .; then
unmanaged_units="${unmanaged_units:+$unmanaged_units }$svc.service"
fi
done
if [ -n "$unmanaged_units" ]; then
echo "refusing to install archied $GW: this host runs units the update does not manage: $unmanaged_units" >&2
echo "a current install.sh writes units for: $GATEWAY_SERVICES. If the installed updater predates this release, replace $BIN_DIR/archie-update-install with scripts/archie-update-install from archied/v$GW; otherwise remove the extra unit, or add it to GATEWAY_SERVICES if this host is meant to run it." >&2
exit 1
if [ "$SOURCE_BUILD" = 1 ]; then
docker build --build-arg "RUNTIME_VERSION=$RT" --iidfile "$tmp/image-id" "$source_dir"
AGENT_IMAGE="$(cat "$tmp/image-id")"
else
AGENT_IMAGE="$(jq -er '.agent_image' "$release_dir/release.json")"
[[ "$AGENT_IMAGE" =~ @sha256:[a-f0-9]{64}$ ]] || { echo "refusing agent update: release has no digest-pinned image" >&2; exit 1; }
docker pull "$AGENT_IMAGE"
fi
fi
if [[ ",$components," == *,daemon,* ]]; then
echo "==> snapshotting the database before the update migrates it"
mkdir -p "$(dirname "$DB_SNAPSHOT")"
if ! go run ./cmd/archie-state-store backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT"; then
echo "refusing to install $GW: could not snapshot the database named by $CONFIG_PATH, so a failed migration could not be recovered" >&2
exit 1
fi
if [ "$AUTO" = 1 ]; then
status="$(curl -fsS --max-time 10 "${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}/health/detailed")"
[ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || { echo "refusing auto-update: daemon has work in flight or cannot report it" >&2; exit 1; }
fi
install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install"
install -m755 "$source_dir/scripts/archie-update-watchdog" "$BIN_DIR/archie-update-watchdog"
install -m755 "$source_dir/scripts/archie-update-check" "$BIN_DIR/archie-update-check"
built=""
if [[ ",$components," == *,daemon,* ]]; then
ldflags="-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary"
ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$GW"
ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$RT"
for cmd in $GATEWAY_BINARIES; do
[ -d "./cmd/$cmd" ] || continue
echo "==> building $cmd $GW"
go build -ldflags "$ldflags" -o "$tmp/$cmd" "./cmd/$cmd"
built="${built:+$built }$cmd"
reported="$("$release_dir/$cmd" -version | awk -v cmd="$cmd" '$1 == cmd {print $2; exit}')"
[ "$reported" = "$GW" ] || { echo "candidate $cmd reports $reported, expected $GW" >&2; exit 1; }
done
echo "==> backing up current binaries"
for cmd in $built; do
if [ -f "$BIN_DIR/$cmd" ]; then
cp -f "$BIN_DIR/$cmd" "$BIN_DIR/$cmd.prev"
else
rm -f "$BIN_DIR/$cmd.prev"
fi
done
echo "==> installing $built"
for cmd in $built; do
install -m755 "$tmp/$cmd" "$BIN_DIR/$cmd"
done
if ! verify_installed_versions "$GW"; then
exit 1
fi
mkdir -p "$(dirname "$DB_SNAPSHOT")"
"$BIN_DIR/archie-state-store" backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT"
built="$GATEWAY_BINARIES archie-update-install archie-update-watchdog archie-update-check"
fi
# Freeze recovery code and every prior file before publishing the intent.
journal="$STATE_DIR/preparing"
rm -rf "$journal"
mkdir -p "$journal/previous"
cp "$release_dir/archie-update-watchdog" "$journal/watchdog"
cp "$observer" "$journal/helper"
if [[ ",$components," == *,agent,* ]]; then
git checkout --quiet --detach "$rt_commit"
echo "==> saving managed worker image $AGENT_IMAGE for rollback"
docker image save --output "$AGENT_IMAGE_BACKUP" "$AGENT_IMAGE"
if [ -f "$AGENT_VERSION_FILE" ]; then
cp -f "$AGENT_VERSION_FILE" "$AGENT_VERSION_BACKUP"
AGENT_VERSION_PREVIOUS_EXISTS=1
else
rm -f "$AGENT_VERSION_BACKUP"
AGENT_VERSION_PREVIOUS_EXISTS=0
fi
echo "==> rebuilding managed worker image $AGENT_IMAGE for runtime $RT"
docker build --build-arg "RUNTIME_VERSION=$RT" --tag "$AGENT_IMAGE" .
version_tmp="$(mktemp "$AGENT_VERSION_FILE.tmp.XXXXXX")"
printf '%s\n' "$RT" > "$version_tmp"
chmod 600 "$version_tmp"
mv -f "$version_tmp" "$AGENT_VERSION_FILE"
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -prepare "$AGENT_IMAGE"
fi
echo "==> handing off to the update watchdog for restart and health verification"
chmod 700 "$journal/watchdog"
for cmd in $built; do
if [ -f "$BIN_DIR/$cmd" ]; then cp -p "$BIN_DIR/$cmd" "$journal/previous/$cmd"; fi
install -m755 "$release_dir/$cmd" "$BIN_DIR/.$cmd.new"
done
if [ -f "$AGENT_VERSION_FILE" ]; then cp -p "$AGENT_VERSION_FILE" "$journal/agent.version"; fi
# No unit/config changes are made: topology migration is explicitly refused.
export ARCHIE_UPDATE_AUTO="$AUTO"
export ARCHIE_UPDATE_STATE_DIR="$STATE_DIR"
export ARCHIE_BIN_DIR="$BIN_DIR" ARCHIE_CONFIG_PATH="$CONFIG_PATH"
export ARCHIE_UPDATE_COMPONENTS="$components" ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" ARCHIE_UPDATE_BINARIES="$built"
export ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT"
export ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT"
export ARCHIE_AGENT_VERSION_FILE="$AGENT_VERSION_FILE" ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT"
export ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET"
# Bash's quoted declarations preserve spaces without interpreting journal values.
while IFS= read -r var; do declare -p "$var"; done < <(compgen -e | grep -E '^ARCHIE_(UPDATE_|BIN_DIR$|CONFIG_PATH$|HEALTH_URL$|AGENT_VERSION_FILE$|DB_SNAPSHOT$)') > "$journal/environment"
printf 'prepared\n' > "$journal/phase"
sync -f "$BIN_DIR"
sync -f "$journal"
mv "$journal" "$STATE_DIR/transaction"
sync -f "$STATE_DIR"
journal="$STATE_DIR/transaction"
# The independent watchdog holds the lock and owns promotion, restart and rollback.
systemd-run --user --unit=archied-update-restart --collect --quiet \
--setenv=ARCHIE_BIN_DIR="$BIN_DIR" \
--setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-}" \
--setenv=ARCHIE_UPDATE_REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" \
--setenv=ARCHIE_UPDATE_CHANNEL="${ARCHIE_UPDATE_CHANNEL:-}" \
--setenv=ARCHIE_UPDATE_CHAT_ID="${ARCHIE_UPDATE_CHAT_ID:-0}" \
--setenv=ARCHIE_UPDATE_THREAD_ID="${ARCHIE_UPDATE_THREAD_ID:-0}" \
--setenv=ARCHIE_UPDATE_COMPONENTS="$components" \
--setenv=ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" \
--setenv=ARCHIE_UPDATE_BINARIES="$built" \
--setenv=ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" \
--setenv=ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT" \
--setenv=ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" \
--setenv=ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT" \
--setenv=ARCHIE_AGENT_IMAGE="$AGENT_IMAGE" \
--setenv=ARCHIE_AGENT_IMAGE_BACKUP="$AGENT_IMAGE_BACKUP" \
--setenv=ARCHIE_AGENT_VERSION_FILE="$AGENT_VERSION_FILE" \
--setenv=ARCHIE_AGENT_VERSION_BACKUP="$AGENT_VERSION_BACKUP" \
--setenv=ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS="${AGENT_VERSION_PREVIOUS_EXISTS:-0}" \
--setenv=ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT" \
--setenv=ARCHIE_CONFIG_PATH="$CONFIG_PATH" \
--setenv=ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET" \
"$BIN_DIR/archie-update-watchdog"
--setenv=ARCHIE_UPDATE_STATE_DIR="$STATE_DIR" "$journal/watchdog"
case "$components" in
daemon)
previous="{\"daemon\":\"$(json_escape "$PREV_GW")\"}"
+116 -134
View File
@@ -7,36 +7,39 @@
# It does not rebuild or reinstall the gateway binary.
set -uo pipefail
BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}"
AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}"
AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}"
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}"
# The daemon exports its own health address (releaseupdate.CommandInstaller);
# this default only covers a hand-driven run, and must stay the address archied
# itself serves /healthz on -- [health] listen, defaulted in
# internal/infrastructure/configuration/defaults.go.
STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}"
journal="$STATE_DIR/transaction"
mkdir -p "$STATE_DIR"
exec 9>"$STATE_DIR/lock"
if [ "${1:-}" = --recover ]; then
flock -n 9 || exit 0 # A live installer/watchdog owns recovery.
else
flock -w 60 9 || { echo "update locked by $(cat "$STATE_DIR/holder")" >&2; exit 1; }
fi
[ -d "$journal" ] || exit 0
printf 'watchdog PID %s\n' "$$" > "$STATE_DIR/holder"
source "$journal/environment"
BIN_DIR="$ARCHIE_BIN_DIR"
CONFIG_PATH="$ARCHIE_CONFIG_PATH"
DB_SNAPSHOT="$ARCHIE_DB_SNAPSHOT"
AGENT_VERSION_FILE="$ARCHIE_AGENT_VERSION_FILE"
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}"
HEALTH_URL="${HEALTH_URL%/}"
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
POLL_INTERVAL_SECONDS=3
COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}"
# Set by archie-update-install: the units to restart, in start order, and the
# binaries it replaced. The defaults keep a hand-driven run on a pre-
# decomposition host working.
UNITS="${ARCHIE_UPDATE_UNITS:-archied}"
BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}"
# The State Store's address, so the start order below can mean "ready" rather than
# "launched". Empty when the config names none or the installed updater predates
# passing one -- waiting is then skipped, which is the previous behaviour.
OBSERVE_SECONDS="${ARCHIE_UPDATE_OBSERVE_SECONDS:-120}"
POLL_INTERVAL_SECONDS="${ARCHIE_UPDATE_POLL_INTERVAL:-3}"
COMPONENTS="$ARCHIE_UPDATE_COMPONENTS"
UNITS="$ARCHIE_UPDATE_UNITS"
BINARIES="$ARCHIE_UPDATE_BINARIES"
STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-}"
STATE_STORE_TIMEOUT_SECONDS="${ARCHIE_UPDATE_STATE_STORE_TIMEOUT:-30}"
phase() {
printf '%s\n' "$1" > "$journal/phase.new" && sync -f "$journal" &&
mv "$journal/phase.new" "$journal/phase" && sync -f "$journal"
}
json_escape() {
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
}
@@ -75,10 +78,22 @@ EOF
mv -f "$tmp" "$REPORT_PATH"
}
healthy() {
local report
report="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 1
[ "$(jq -r '.version' <<< "$report")" = "$ARCHIE_UPDATE_INSTALLED_GATEWAY" ] &&
[ "$(jq -r '.status' <<< "$report")" = ok ]
}
wait_healthy() {
local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS))
while [ "$SECONDS" -lt "$deadline" ]; do
if curl -sf -o /dev/null "$HEALTH_URL/healthz"; then
if healthy; then
deadline=$((SECONDS + OBSERVE_SECONDS))
while [ "$SECONDS" -lt "$deadline" ]; do
sleep "$POLL_INTERVAL_SECONDS"
healthy || return 1
done
return 0
fi
sleep "$POLL_INTERVAL_SECONDS"
@@ -86,20 +101,7 @@ wait_healthy() {
return 1
}
# manage_unit runs one systemctl action. archied is required: this whole
# mechanism exists to restart it, and a failed stop must abort the rollback
# rather than let it race a live process over the binary and the database.
# The sibling processes are best-effort, because archie-update-install has
# already refused the update if a unit for one of them was missing, and a
# probe that guesses wrong must not silently skip the required stop.
manage_unit() {
local action="$1" unit="$2"
if [ "$unit" = archied ]; then
systemctl --user "$action" "$unit.service"
return
fi
systemctl --user "$action" "$unit.service" || true
}
manage_unit() { systemctl --user "$1" "$2.service"; }
# port_accepts reports whether host:port accepts a connection. bash's /dev/tcp
# costs no process and no dependency, and accepting is exactly the signal this
@@ -109,21 +111,7 @@ port_accepts() {
(exec 3<>"/dev/tcp/${1%:*}/${1##*:}") 2>/dev/null
}
# wait_for_state_store blocks until the State Store accepts connections, or the
# budget runs out.
#
# Ordering the units is not enough on its own: `systemctl --user start` returns
# once the process is forked, not once its port accepts. Measured on a 1.37.1 ->
# 1.38.0 update, the State Store's unit reported started while it was still
# binding, archied was started next, dialled a closed port and exited 1, and only
# systemd's Restart=on-failure brought it back seven seconds later. A host whose
# unit lacks that restart policy gets a failed update behind a healthy-looking
# "build and install succeeded" line.
#
# A timeout does NOT fail the update. If the State Store never comes up, archied
# fails closed on its own and wait_healthy rolls the release back -- waiting is an
# optimisation for the ordinary case, and losing the race must stay a slower path
# to the same place rather than a second failure mode.
# Wait for the dependency before starting its callers.
wait_for_state_store() {
[ -n "$STATE_STORE_TARGET" ] || {
echo "==> no State Store target known; starting the rest without waiting for it" >&2
@@ -169,39 +157,70 @@ stop_units() {
return $rc
}
# rollback_gateway restores every binary the install replaced. A binary with
# no .prev was introduced by the failed release, so it is removed rather than
# left behind pointing at processes the restored release does not run.
rollback_gateway() {
local cmd rc=0
rollback_files() {
local cmd
for cmd in $BINARIES; do
if [ -f "$BIN_DIR/$cmd.prev" ]; then
cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1
if [ -f "$journal/previous/$cmd" ]; then
cp -p "$journal/previous/$cmd" "$BIN_DIR/.$cmd.rollback" &&
mv -f "$BIN_DIR/.$cmd.rollback" "$BIN_DIR/$cmd" || return 1
else
rm -f "$BIN_DIR/$cmd" || rc=1
rm -f "$BIN_DIR/$cmd" || return 1
fi
rm -f "$BIN_DIR/.$cmd.new"
done
return $rc
if [[ ",$COMPONENTS," == *,agent,* ]]; then
if [ -f "$journal/agent.version" ]; then
cp -p "$journal/agent.version" "$AGENT_VERSION_FILE.new" &&
mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1
else
rm -f "$AGENT_VERSION_FILE" || return 1
fi
fi
sync -f "$BIN_DIR"
}
rollback_agent_image() {
if [ ! -f "$AGENT_IMAGE_BACKUP" ]; then
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
return 1
fi
docker image load --input "$AGENT_IMAGE_BACKUP"
finish() {
phase "$1" || return 1
# A terminal journal is harmless if power fails before removal.
rm -rf "$STATE_DIR/finished"
mv "$journal" "$STATE_DIR/finished" && sync -f "$STATE_DIR" || return 1
rm -rf "$STATE_DIR/finished"
}
# The sidecar describes the image restored above. Restore its prior state in
# the same rollback path; a first install has no prior file and must remove the
# newly-created metadata rather than leave a false installed version behind.
rollback_agent_version() {
if [ "$AGENT_VERSION_PREVIOUS_EXISTS" = 1 ]; then
[ -f "$AGENT_VERSION_BACKUP" ] || return 1
cp -f "$AGENT_VERSION_BACKUP" "$AGENT_VERSION_FILE"
else
rm -f "$AGENT_VERSION_FILE"
rollback() {
phase rolling-back || return 1
stop_units && rollback_files || return 1
if [ -f "$journal/image.json" ]; then
manage_unit start archie-state-store && wait_for_state_store &&
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -rollback || return 1
fi
phase rolled-back || return 1
start_units || return 1
write_report failed true || return 1
finish rolled-back
}
promote() {
if [ "${ARCHIE_UPDATE_AUTO:-0}" = 1 ]; then
local status
status="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 2
[ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || return 2
fi
phase switching || return 1
stop_units || return 1
for cmd in $BINARIES; do
mv -f "$BIN_DIR/.$cmd.new" "$BIN_DIR/$cmd" || return 1
done
if [[ ",$COMPONENTS," == *,agent,* ]]; then
printf '%s\n' "$ARCHIE_UPDATE_INSTALLED_RUNTIME" > "$AGENT_VERSION_FILE.new" &&
mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1
fi
sync -f "$BIN_DIR" || return 1
if [ -f "$journal/image.json" ]; then
manage_unit start archie-state-store && wait_for_state_store &&
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" || return 1
fi
phase observing && start_units && wait_healthy
}
# explain_database_recovery states what a daemon rollback does not do. The
@@ -214,62 +233,25 @@ explain_database_recovery() {
echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2
}
start_units
if wait_healthy; then
echo "==> new version is healthy"
write_report "passed" false
case "$(cat "$journal/phase")" in
committed) start_units && finish committed; exit $? ;;
rolled-back) start_units && finish rolled-back; exit $? ;;
esac
if [ -z "${1:-}" ] && promote; then
write_report passed false && finish committed
exit $?
fi
if [ "$(cat "$journal/phase")" = prepared ]; then
echo "update was not promoted; leaving running work alone" >&2
for cmd in $BINARIES; do rm -f "$BIN_DIR/.$cmd.new"; done
write_report failed false && finish rolled-back || exit 1
if [ "${1:-}" = --boot-recover ]; then start_units; exit $?; fi
exit 1
fi
echo "==> restoring interrupted or unhealthy update" >&2
if rollback; then
if [[ ",$COMPONENTS," == *,daemon,* ]]; then explain_database_recovery; fi
exit 0
fi
echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components"
rolled_back=false
# Stop before touching the executable. The service uses Restart=on-failure in
# the supported unit, so copying rollback state while a crash-loop child is
# alive can execute the new binary after its files have been replaced.
if stop_units; then
case "$COMPONENTS" in
daemon)
if [ -f "$BIN_DIR/archied.prev" ]; then
if rollback_gateway; then
rolled_back=true
fi
else
echo "==> no backup for archied; cannot roll it back automatically" >&2
fi
;;
agent)
if [ -f "$AGENT_IMAGE_BACKUP" ]; then
if rollback_agent_image && rollback_agent_version; then
rolled_back=true
fi
else
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
fi
;;
daemon,agent)
# Preserve the old all-or-nothing rollback guarantee: never restore one
# component and leave the other on the failed release.
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ]; then
if rollback_agent_image && rollback_agent_version && rollback_gateway; then
rolled_back=true
fi
else
echo "==> not all changed-component backups exist; cannot roll back automatically" >&2
fi
;;
*)
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
;;
esac
else
echo "==> could not stop every unit; refusing rollback while one may be running" >&2
fi
if [ "$rolled_back" = true ]; then
start_units
fi
if [[ ",$COMPONENTS," == *,daemon,* ]]; then
explain_database_recovery
fi
write_report "failed" "$rolled_back"
echo "rollback incomplete; retained journal: $journal" >&2
exit 1