Files
archie/internal/releaseupdate/verify_test.go
T
sam 2873cbba82 fix(update): verify an update took effect instead of trusting the report (#530)
A pending update report used to be relayed verbatim. It is entirely
self-reported: scripts/archie-update-watchdog decides "passed" from
`curl -sf $HEALTH_URL/healthz`, which only proves that *a* daemon is
answering, then copies the version the installer *intended*
(ARCHIE_UPDATE_INSTALLED_GATEWAY) into the report. If the new binary
never made it into service, the previous daemon answers the probe just
as happily, so the message read "Update complete: daemon 1.9.8 -> 1.9.0.
Health check passed - archie-core is healthy on the new version" about a
version that was not running. That shipped: /update reported success and
the next task hit the exact bug the update was meant to fix (#468).

Report.Verify now checks the claims against the versions components
report for *themselves* -- archied's compiled-in gatewayVersion, which no
installer can talk it out of -- and partitions every claimed component
into Confirmed, Drift or Unverified.

What changed in MEANING:

- A passed health check used to mean "the new version is running". It
  now means only "something answered"; the version claim is a separate,
  separately-checked assertion.
- "Update complete ... healthy on the new version" used to be the only
  success wording. There are now three outcomes, because collapsing them
  is how the false success happened: confirmed, unconfirmed ("I could
  not confirm which version is actually running"), and contradicted
  ("Update did NOT take effect").
- An unverifiable component is NOT a failing one. Absent, empty,
  "unknown" (the watchdog's own default) and "dev" (an unstamped build)
  all mean "we could not check", never "it is wrong" -- treating them as
  drift would turn healthy updates whose bookkeeping went missing into
  reported failures.

Verification covers the daemon only, deliberately. archied's compiled-in
runtimeVersion records what its own pipeline stamped, not what the agent
container runs, so vouching for the agent would be the same kind of
guess this change exists to remove. The agent is therefore Unverified on
every successful update, and the success message names it as unchecked
rather than letting the confirmed daemon silently vouch for it.

Drift detail is attached per component: one component can sit on the
version the update meant to replace while another is on some third
version, and a single explanatory sentence would state a falsehood about
one of them.

Regression tests were confirmed to fail against the pre-fix code,
reproducing the exact false-success string, and each new guard was
mutation-tested. The drift-ordering test's component count is
load-bearing and documented: Go rotates small maps rather than permuting
them, so 8 components let an unsorted implementation pass ~12% of the
time while 9 fails it outright.

Refs #522, #468
2026-08-21 21:42:48 +10:00

238 lines
8.4 KiB
Go

package releaseupdate_test
import (
"fmt"
"sort"
"testing"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
const (
daemon = releaseupdate.ComponentDaemon
agent = releaseupdate.ComponentAgent
)
// TestVerifyDetectsDrift covers the false-success bug this
// verification exists for: an installer reports a version it never actually
// managed to put into service, and nothing contradicts it because the health
// check only proves *a* daemon is answering -- possibly the old one.
func TestVerifyDetectsDrift(t *testing.T) {
tests := []struct {
name string
previous map[string]string
installed map[string]string
running map[string]string
want []releaseupdate.VersionDrift
}{
{
name: "claim matches running",
installed: map[string]string{daemon: "1.9.0"},
running: map[string]string{daemon: "1.9.0"},
want: nil,
},
{
name: "claim contradicted by the version it replaced",
previous: map[string]string{daemon: "1.9.8"},
installed: map[string]string{daemon: "1.9.0"},
running: map[string]string{daemon: "1.9.8"},
want: []releaseupdate.VersionDrift{
{ID: daemon, Claimed: "1.9.0", Running: "1.9.8", RunningIsPrevious: true},
},
},
{
name: "running is neither installed nor previous",
previous: map[string]string{daemon: "1.9.8"},
installed: map[string]string{daemon: "1.9.0"},
running: map[string]string{daemon: "1.2.3"},
want: []releaseupdate.VersionDrift{
{ID: daemon, Claimed: "1.9.0", Running: "1.2.3", RunningIsPrevious: false},
},
},
{
name: "unknown previous cannot make running the previous version",
previous: map[string]string{daemon: releaseupdate.VersionUnknown},
installed: map[string]string{daemon: "1.9.0"},
running: map[string]string{daemon: "1.9.8"},
want: []releaseupdate.VersionDrift{
{ID: daemon, Claimed: "1.9.0", Running: "1.9.8", RunningIsPrevious: false},
},
},
{
name: "component with no running version is unverifiable, not wrong",
installed: map[string]string{daemon: "1.9.0", agent: "1.9.0"},
running: map[string]string{daemon: "1.9.0"},
want: nil,
},
{
name: "empty running version is unverifiable, not wrong",
installed: map[string]string{daemon: "1.9.0"},
running: map[string]string{daemon: ""},
want: nil,
},
{
name: "unknown running version is unverifiable, not wrong",
installed: map[string]string{daemon: "1.9.0"},
running: map[string]string{daemon: releaseupdate.VersionUnknown},
want: nil,
},
{
name: "unknown claim asserts nothing to contradict",
installed: map[string]string{daemon: releaseupdate.VersionUnknown},
running: map[string]string{daemon: "1.9.8"},
want: nil,
},
{
name: "empty claim asserts nothing to contradict",
installed: map[string]string{daemon: ""},
running: map[string]string{daemon: "1.9.8"},
want: nil,
},
{
name: "nothing running verifies nothing",
installed: map[string]string{daemon: "1.9.0"},
running: nil,
want: nil,
},
{
name: "an unstamped running build is unverifiable, not wrong",
installed: map[string]string{daemon: "1.9.0"},
running: map[string]string{daemon: releaseupdate.VersionDev},
want: nil,
},
{
name: "an unstamped claim asserts nothing to contradict",
installed: map[string]string{daemon: releaseupdate.VersionDev},
running: map[string]string{daemon: "1.9.0"},
want: nil,
},
{
name: "an upper-case tag prefix does not manufacture drift",
installed: map[string]string{daemon: "V1.9.0"},
running: map[string]string{daemon: "1.9.0"},
want: nil,
},
{
name: "a tag-shaped claim matches the stamped version it names",
installed: map[string]string{daemon: "v1.9.0"},
running: map[string]string{daemon: "1.9.0"},
want: nil,
},
{
name: "surrounding whitespace does not manufacture drift",
installed: map[string]string{daemon: " 1.9.0\n"},
running: map[string]string{daemon: "1.9.0"},
want: nil,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
report := releaseupdate.Report{Previous: test.previous, Installed: test.installed}
got := report.Verify(test.running).Drift
if len(got) != len(test.want) {
t.Fatalf("Verify().Drift = %#v, want %#v", got, test.want)
}
for i := range test.want {
if got[i] != test.want[i] {
t.Errorf("Verify().Drift[%d] = %#v, want %#v", i, got[i], test.want[i])
}
}
})
}
}
// TestVerifyPartitionsEveryClaim proves each claimed component lands in
// exactly one of Confirmed/Drift/Unverified. Callers word their message from
// which set is non-empty, so a component silently falling into none of them
// (or into two) is what would let an unchecked claim be reported as a
// checked one.
func TestVerifyPartitionsEveryClaim(t *testing.T) {
tests := []struct {
name string
claimed string
running string
wantConfirmed bool
wantDrift bool
wantUnverified bool
}{
{name: "agreeing versions confirm", claimed: "1.9.0", running: "1.9.0", wantConfirmed: true},
{name: "disagreeing versions drift", claimed: "1.9.0", running: "1.9.8", wantDrift: true},
{name: "unrecorded claim is unverified", claimed: releaseupdate.VersionUnknown, running: "1.9.8", wantUnverified: true},
{name: "unstamped runner is unverified", claimed: "1.9.0", running: releaseupdate.VersionDev, wantUnverified: true},
{name: "absent runner is unverified", claimed: "1.9.0", running: "", wantUnverified: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
report := releaseupdate.Report{Installed: map[string]string{daemon: test.claimed}}
got := report.Verify(map[string]string{daemon: test.running})
total := len(got.Confirmed) + len(got.Drift) + len(got.Unverified)
if total != 1 {
t.Fatalf("claim landed in %d sets, want exactly 1: %#v", total, got)
}
if (len(got.Confirmed) == 1) != test.wantConfirmed {
t.Errorf("Confirmed = %v, want confirmed=%v", got.Confirmed, test.wantConfirmed)
}
if (len(got.Drift) == 1) != test.wantDrift {
t.Errorf("Drift = %#v, want drift=%v", got.Drift, test.wantDrift)
}
if (len(got.Unverified) == 1) != test.wantUnverified {
t.Errorf("Unverified = %v, want unverified=%v", got.Unverified, test.wantUnverified)
}
})
}
}
// TestVerifyIgnoresUnclaimedRunningComponents proves a component
// that is running but was never part of the update is not treated as drift:
// the claim set is what is under test, not the inventory.
func TestVerifyIgnoresUnclaimedRunningComponents(t *testing.T) {
report := releaseupdate.Report{Installed: map[string]string{daemon: "1.9.0"}}
running := map[string]string{daemon: "1.9.0", agent: "0.0.1"}
if got := report.Verify(running).Drift; len(got) != 0 {
t.Fatalf("Verify().Drift = %#v, want no drift", got)
}
}
// TestVerifyOrdersDriftStably guards the sort, which callers rely on
// to render a stable message.
//
// The component count is load-bearing and is NOT about permutations. Go
// randomizes a small map's iteration start offset rather than its order, so
// up to 8 keys in one bucket come out as a rotation -- an unsorted
// implementation still emits sorted output about 1/n of the time, not 1/n!.
// Measured against this code with the sort removed: 2 components pass 87% of
// the time and 8 pass 12%, while 9 crosses into bucket-order randomization
// and passes 0 times in 200,000. Nine is therefore the first count that
// fails an unsorted implementation on a single attempt; the repeats are
// belt-and-braces. Do not reduce either number.
func TestVerifyOrdersDriftStably(t *testing.T) {
const components = 9
installed := make(map[string]string, components)
running := make(map[string]string, components)
want := make([]string, 0, components)
for i := range components {
id := fmt.Sprintf("component-%d", i)
installed[id] = "2.0.0"
running[id] = "1.0.0"
want = append(want, id)
}
sort.Strings(want)
report := releaseupdate.Report{Installed: installed}
for attempt := range 25 {
drift := report.Verify(running).Drift
if len(drift) != components {
t.Fatalf("attempt %d: Verify().Drift returned %d drifts, want %d", attempt, len(drift), components)
}
for i, id := range want {
if drift[i].ID != id {
t.Fatalf("attempt %d: drift[%d].ID = %q, want %q (order is not stable)", attempt, i, drift[i].ID, id)
}
}
}
}