refactor(archied): move the update relay to the process that owns updates

The dashboard's Run read the watchdog's phase-2 report off local disk and
put it on the activity stream. That is a host-local file read in the process
that is moving off the host: the watchdog restarts archied and leaves its
verdict beside archied, so archied is what can read it.

The daemon now polls and relays it, publishing the same event through the
same bus, so it persists with every other event and reaches whichever
dashboard is watching. The event builder moves to releaseupdate, which owns
Report and its verification; the dashboard keeps only what it needs to tell
an installer where to write.

The startup poll stays: the watchdog writes its result only once the process
answers /healthz, so a single read at boot can always miss the report
describing that very boot.

persistAndBroadcastEvents takes the daemon context and detaches it for the
insert, which is what it always did with context.Background() -- the last
events of a run are the ones an operator reads afterwards.
This commit is contained in:
sam committed 2026-09-09 23:55:20 +10:00
1 parent 801ac1570c
commit 30228f3449
9 files changed
+266 -197

No files matched your search

+6 -2
View File
@@ -366,7 +366,7 @@ func (b *boot) loadCatalog(ctx context.Context, cfgPath string) {
// setupObservability builds the event bus, channel manager and dashboard
// server. Every event is logged to SQLite (stamped with its row id) and
// then fanned out to live dashboard connections.
func (b *boot) setupObservability() {
func (b *boot) setupObservability(ctx context.Context) {
cfg, log := b.cfg, b.log
bus := events.NewBus()
b.bus = bus
@@ -388,6 +388,10 @@ func (b *boot) setupObservability() {
}
b.web = &webui.Server{Store: b.stateStore, Log: log.With("component", "webui"), LogFeed: b.logFeed, TaskLogs: b.taskLogs, Cfg: config.NewHolder(cfg), Channels: b.channelManager, Events: bus}
b.web.UpdateReportPath = updateReportPath(cfg.WorkDir, "webui")
// The watchdog leaves its verdict in a file on this host, so the daemon
// reads it and publishes the outcome as an event; the dashboard renders
// what it receives, wherever it runs (archie-core-8cda.5.4).
b.startUpdateRelay(ctx, b.web.UpdateReportPath)
if (cfg.Chat.Telegram.Token != (secret.SecretRef{}) || cfg.Chat.Telegram.TokenEnv != "") && len(cfg.Chat.Telegram.AllowedUserIDs) > 0 {
b.web.TelegramUpdateReportPath = updateReportPath(cfg.WorkDir, cfg.BotUser)
b.web.TelegramUpdateChatID = cfg.Chat.Telegram.AllowedUserIDs[0]
@@ -403,7 +407,7 @@ func (b *boot) setupObservability() {
}
b.wireWebStoreSurfaces()
sink := bus.Subscribe(256)
go persistAndBroadcastEvents(sink, b.stateStore, b.web, log)
go persistAndBroadcastEvents(ctx, sink, b.stateStore, b.web, log)
}
// wireWebStoreSurfaces attaches the dashboard's optional storage surfaces.
+10 -3
View File
@@ -399,7 +399,7 @@ func Run() int { //nolint:cyclop // the composition root's setup sequence is del
return 0
}
b.loadCatalog(ctx, args.cfgPath)
b.setupObservability()
b.setupObservability(ctx)
if err := b.setupBackends(ctx); err != nil {
return 1
@@ -509,10 +509,17 @@ func recordBootOverlayError(p *atomic.Pointer[string], log *slog.Logger, err err
log.Error(msg, args...)
}
func persistAndBroadcastEvents(sink *events.Sub, st store.TaskStore, web *webui.Server, log *slog.Logger) {
// persistAndBroadcastEvents drains the bus until it closes, giving each
// event an ID from the store before fanning it out.
//
// The insert deliberately outlives ctx: this loop ends when the bus closes,
// which is part of shutdown, and the last events of a run are exactly the
// ones an operator wants to read afterwards.
func persistAndBroadcastEvents(ctx context.Context, sink *events.Sub, st store.TaskStore, web *webui.Server, log *slog.Logger) {
ctx = context.WithoutCancel(ctx)
for e := range sink.C {
if e.ID == 0 {
id, err := st.InsertEvent(context.Background(), e)
id, err := st.InsertEvent(ctx, e)
if err != nil {
log.Error("event sink insert failed", "err", err)
continue
@@ -93,7 +93,7 @@ func TestSetupObservabilityRoutesTaskStoreThroughStateStore(t *testing.T) {
taskLogs: logging.NewTaskRegistry(filepath.Join(t.TempDir(), "logs"), logFeed, logging.TaskSinkOptions{}),
}
t.Cleanup(b.cleanup)
b.setupObservability()
b.setupObservability(t.Context())
if b.web.Store != storeB {
t.Fatalf("webui Store = %p, want b.stateStore (%p); dashboard task store did not route through the State Store adapter", b.web.Store, storeB)
+63
View File
@@ -0,0 +1,63 @@
package archied
import (
"context"
"time"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
// updateReportPollInterval closes the watchdog/startup race: the watchdog
// writes its result only once this process answers /healthz, so a read at
// boot alone can always miss the report describing that very boot.
const updateReportPollInterval = 2 * time.Second
// startUpdateRelay publishes the update watchdog's phase-2 outcome onto the
// activity stream.
//
// The daemon owns this because the daemon owns the update: the watchdog
// restarts archied and leaves its verdict in a file on archied's host, so
// only archied can read it. It previously ran inside the dashboard's own
// Run, which put a host-local file read in the process that is moving off
// the host (archie-core-8cda.5.4). The report reaches an operator the same
// way it always did, as an event -- persisted with every other event and
// delivered to whichever dashboard is watching.
//
// A report is cleared as soon as it is handled, relayed or unreadable, so a
// later boot never retries it and two boots never race to relay one outcome.
func (b *boot) startUpdateRelay(ctx context.Context, path string) {
if path == "" || b.bus == nil {
return
}
b.relayUpdateReport(path)
go func() {
ticker := time.NewTicker(updateReportPollInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
b.relayUpdateReport(path)
}
}
}()
}
func (b *boot) relayUpdateReport(path string) {
report, found, err := releaseupdate.ReadPendingReport(path)
if err != nil {
b.log.Warn("read pending update report failed; discarding it", "err", err)
if clearErr := releaseupdate.ClearPendingReport(path); clearErr != nil {
b.log.Warn("clear unreadable update report failed", "err", clearErr)
}
return
}
if !found {
return
}
b.publishEvent(releaseupdate.ReportEvent(report, daemonRunningVersions(b.agentStatus)))
if err := releaseupdate.ClearPendingReport(path); err != nil {
b.log.Warn("clear pending update report failed", "err", err)
}
}
+121
View File
@@ -0,0 +1,121 @@
package archied
import (
"log/slog"
"os"
"path/filepath"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
// relayBoot is the daemon reduced to what the relay needs: a bus to publish
// on and a logger.
func relayBoot(t *testing.T) (*boot, *events.Sub) {
t.Helper()
bus := events.NewBus()
t.Cleanup(bus.Close)
sub := bus.Subscribe(4)
return &boot{bus: bus, log: slog.New(slog.DiscardHandler)}, sub
}
func waitForEvent(t *testing.T, sub *events.Sub) (events.Event, bool) {
t.Helper()
select {
case e := <-sub.C:
return e, true
case <-time.After(2 * time.Second):
return events.Event{}, false
}
}
// TestRelayUpdateReportPublishesAndClears: the watchdog leaves its verdict in
// a file on the daemon's host, so the daemon reads it and puts the outcome on
// the activity stream. Relaying twice would tell the operator the same update
// finished twice, so the report is cleared as soon as it is handled
// (archie-core-nln7, rehomed by archie-core-8cda.5.4).
func TestRelayUpdateReportPublishesAndClears(t *testing.T) {
b, sub := relayBoot(t)
path := filepath.Join(t.TempDir(), "update-report.json")
report := releaseupdate.Report{
Channel: "webui",
Previous: map[string]string{releaseupdate.ComponentDaemon: "1.0.0"},
Installed: map[string]string{releaseupdate.ComponentDaemon: "1.1.0"},
HealthCheck: "passed",
}
if err := releaseupdate.WritePendingReport(path, report); err != nil {
t.Fatal(err)
}
b.relayUpdateReport(path)
e, ok := waitForEvent(t, sub)
if !ok {
t.Fatal("no event published for a pending update report")
}
if e.Kind != events.KindUpdateReport {
t.Errorf("kind = %q, want %q", e.Kind, events.KindUpdateReport)
}
if e.Data["health_check"] != "passed" {
t.Errorf("health_check = %v, want passed", e.Data["health_check"])
}
if _, found, err := releaseupdate.ReadPendingReport(path); err != nil || found {
t.Errorf("report still present after relaying: found=%v err=%v", found, err)
}
// A second pass has nothing left to relay.
b.relayUpdateReport(path)
if e, ok := waitForEventQuickly(sub); ok {
t.Fatalf("a cleared report was relayed again: %+v", e)
}
}
func waitForEventQuickly(sub *events.Sub) (events.Event, bool) {
select {
case e := <-sub.C:
return e, true
case <-time.After(100 * time.Millisecond):
return events.Event{}, false
}
}
// TestRelayUpdateReportClearsUnreadableReport: a corrupt report must not jam
// every future boot behind it.
func TestRelayUpdateReportClearsUnreadableReport(t *testing.T) {
b, _ := relayBoot(t)
path := filepath.Join(t.TempDir(), "update-report.json")
if err := os.WriteFile(path, []byte("{not valid json"), 0o600); err != nil {
t.Fatal(err)
}
b.relayUpdateReport(path)
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("corrupt report file still exists after relaying: err = %v", err)
}
}
func TestRelayUpdateReportWithNoFileIsSilent(t *testing.T) {
b, sub := relayBoot(t)
b.relayUpdateReport(filepath.Join(t.TempDir(), "does-not-exist.json"))
if e, ok := waitForEventQuickly(sub); ok {
t.Fatalf("published %+v with no report to relay", e)
}
}
// TestStartUpdateRelayWithoutAPathDoesNothing: a deployment with no phase-2
// reporting configured must not start a poller for a file that will never
// exist.
func TestStartUpdateRelayWithoutAPathDoesNothing(t *testing.T) {
b, sub := relayBoot(t)
b.startUpdateRelay(t.Context(), "")
if e, ok := waitForEventQuickly(sub); ok {
t.Fatalf("published %+v with no report path configured", e)
}
}
@@ -1,66 +1,20 @@
package webui
package releaseupdate
import (
"context"
"fmt"
"sort"
"strings"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
// reportPendingUpdate checks for a phase-2 outcome left by the update
// watchdog for a dashboard-initiated install and relays it onto the
// activity stream, mirroring internal/channels/telegram/update.go's
// reportPendingUpdate for the Telegram channel. Called once on every daemon
// boot (Server.Run), so it fires whether this is the restart the update
// itself triggered or a later, unrelated restart.
//
// A report is cleared as soon as it's handled -- relayed, or found
// unreadable -- so a later boot never retries it and two boots never race
// to relay the same outcome.
func (s *Server) reportPendingUpdate(ctx context.Context) {
s.updateReportMu.Lock()
defer s.updateReportMu.Unlock()
if s.UpdateReportPath == "" {
return
}
report, found, err := releaseupdate.ReadPendingReport(s.UpdateReportPath)
if err != nil {
s.logf("read pending update report failed; discarding it", "err", err)
if clearErr := releaseupdate.ClearPendingReport(s.UpdateReportPath); clearErr != nil {
s.logf("clear unreadable update report failed", "err", clearErr)
}
return
}
if !found {
return
}
s.emit(ctx, updateReportEvent(report, s.runningVersions()))
if err := releaseupdate.ClearPendingReport(s.UpdateReportPath); err != nil {
s.logf("clear pending update report failed", "err", err)
}
}
// runningVersions returns the versions components report for themselves, or
// nil when the composition root wired none -- see the identical seam on
// internal/channels/telegram.Gateway. Nil is not a no-op: with nothing to
// check against, every claim comes back Unverified rather than Confirmed.
func (s *Server) runningVersions() map[string]string {
if s.RunningVersions == nil {
return nil
}
return s.RunningVersions()
}
// updateReportEvent turns a phase-2 Report, plus its verification against
// ReportEvent turns a phase-2 Report, plus its verification against
// the versions actually running, into the activity-stream event the
// dashboard renders. Unlike Telegram's formatPendingReport, this ships the
// structured Verification rather than pre-rendered prose, so the frontend
// controls presentation -- the two channels' operators read the same facts
// through different UIs, not necessarily the same sentence.
func updateReportEvent(report releaseupdate.Report, running map[string]string) events.Event {
func ReportEvent(report Report, running map[string]string) events.Event {
verification := report.Verify(running)
drift := make([]map[string]any, 0, len(verification.Drift))
for _, d := range verification.Drift {
@@ -73,7 +27,7 @@ func updateReportEvent(report releaseupdate.Report, running map[string]string) e
}
return events.Event{
Kind: events.KindUpdateReport,
Detail: summarizeUpdateReport(report, verification),
Detail: summarizeReport(report, verification),
Data: map[string]any{
"health_check": report.HealthCheck,
"rolled_back": report.RolledBack,
@@ -86,14 +40,14 @@ func updateReportEvent(report releaseupdate.Report, running map[string]string) e
}
}
// summarizeUpdateReport is the one-line summary shown in the dashboard's
// summarizeReport is the one-line summary shown in the dashboard's
// activity feed, which renders event.Detail generically for every event
// kind (see ui/src/dashboard/dashboard.js activityRow). It condenses the
// same facts internal/channels/telegram/update.go's formatPendingReport
// spells out at length -- not a byte-identical wording, since the two
// surfaces render differently, but the same four outcomes: confirmed,
// drifted, unverified, or rolled back.
func summarizeUpdateReport(report releaseupdate.Report, verification releaseupdate.Verification) string {
func summarizeReport(report Report, verification Verification) string {
if report.RolledBack {
return "Update failed its health check and was rolled back to the previous version."
}
@@ -0,0 +1,57 @@
package releaseupdate
import (
"strings"
"testing"
"github.com/samcharles93/archie-core/internal/events"
)
// TestReportEventDistinguishesDriftFromConfirmed is the false-success
// regression the whole verification mechanism exists to catch: a passed
// health probe alone must never be worded as a confirmed version.
func TestReportEventDistinguishesDriftFromConfirmed(t *testing.T) {
report := Report{
HealthCheck: "passed",
Previous: map[string]string{ComponentDaemon: "1.0.0"},
Installed: map[string]string{ComponentDaemon: "1.1.0"},
}
drifted := ReportEvent(report, map[string]string{ComponentDaemon: "1.0.0"})
if drifted.Kind != events.KindUpdateReport {
t.Fatalf("kind = %q, want %q", drifted.Kind, events.KindUpdateReport)
}
if !strings.Contains(drifted.Detail, "did NOT take effect") {
t.Errorf("summary = %q, want it to flag drift rather than claim success", drifted.Detail)
}
if strings.Contains(drifted.Detail, "confirmed running") {
t.Errorf("summary = %q, must not claim confirmed running when drift was detected", drifted.Detail)
}
confirmed := ReportEvent(report, map[string]string{ComponentDaemon: "1.1.0"})
if !strings.Contains(confirmed.Detail, "confirmed running") {
t.Errorf("summary = %q, want it to confirm the running version", confirmed.Detail)
}
ids, _ := confirmed.Data["confirmed"].([]string)
if len(ids) != 1 || ids[0] != ComponentDaemon {
t.Errorf("confirmed = %#v, want [%q]", confirmed.Data["confirmed"], ComponentDaemon)
}
}
// TestReportEventRolledBackIsNeverWordedAsSuccess.
func TestReportEventRolledBackIsNeverWordedAsSuccess(t *testing.T) {
report := Report{
HealthCheck: "failed",
RolledBack: true,
Previous: map[string]string{ComponentDaemon: "1.0.0"},
Installed: map[string]string{ComponentDaemon: "1.1.0"},
}
e := ReportEvent(report, map[string]string{ComponentDaemon: "1.0.0"})
if !strings.Contains(e.Detail, "rolled back") {
t.Errorf("summary = %q, want it to report the rollback", e.Detail)
}
if e.Data["rolled_back"] != true {
t.Errorf("rolled_back = %v, want true", e.Data["rolled_back"])
}
}
+2 -21
View File
@@ -213,9 +213,8 @@ type Server struct {
// When Cfg is present, Cfg.Get().Web.TrustForwardedHeaders is also checked.
TrustForwardedHeaders bool
mu sync.Mutex
conns map[chan events.Event]struct{}
updateReportMu sync.Mutex
mu sync.Mutex
conns map[chan events.Event]struct{}
}
func (s *Server) trustForwardedHeaders() bool {
@@ -390,8 +389,6 @@ func trimLeadingSlash(p string) string {
// Run serves until ctx ends.
func (s *Server) Run(ctx context.Context, listen string) error {
s.reportPendingUpdate(ctx)
go s.watchPendingUpdate(ctx)
srv := &http.Server{Addr: listen, Handler: s.Handler(), ReadHeaderTimeout: 5 * time.Second}
go func() {
<-ctx.Done()
@@ -406,22 +403,6 @@ func (s *Server) Run(ctx context.Context, listen string) error {
return nil
}
// watchPendingUpdate closes the watchdog/startup race: the watchdog writes
// the health result only after this process answers /healthz, so a startup
// read alone can always miss a report for the very boot it describes.
func (s *Server) watchPendingUpdate(ctx context.Context) {
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s.reportPendingUpdate(ctx)
}
}
}
func writeJSON(w http.ResponseWriter, v any) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(v)
-118
View File
@@ -6,131 +6,13 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/gateway"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
// TestReportPendingUpdateEmitsAndClearsReport is the webui counterpart of
// telegram's TestReportPendingUpdateSendsAndClearsReport: a dashboard-
// initiated install must produce a phase-2 report that reaches the
// dashboard, and must never be relayed twice (archie-core-nln7).
func TestReportPendingUpdateEmitsAndClearsReport(t *testing.T) {
s := newTestServer(t)
path := filepath.Join(t.TempDir(), "update-report.json")
report := releaseupdate.Report{
Channel: "webui",
Previous: map[string]string{releaseupdate.ComponentDaemon: "1.0.0"},
Installed: map[string]string{releaseupdate.ComponentDaemon: "1.1.0"},
HealthCheck: "passed",
}
if err := releaseupdate.WritePendingReport(path, report); err != nil {
t.Fatal(err)
}
s.UpdateReportPath = path
s.RunningVersions = func() map[string]string {
return map[string]string{releaseupdate.ComponentDaemon: "1.1.0"}
}
ctx := context.Background()
s.reportPendingUpdate(ctx)
evs, err := s.Store.EventsSince(ctx, 0, 10)
if err != nil {
t.Fatal(err)
}
if len(evs) != 1 {
t.Fatalf("events = %#v, want exactly one", evs)
}
if evs[0].Kind != events.KindUpdateReport {
t.Errorf("kind = %q, want %q", evs[0].Kind, events.KindUpdateReport)
}
confirmed, _ := evs[0].Data["confirmed"].([]any)
if len(confirmed) != 1 || confirmed[0] != releaseupdate.ComponentDaemon {
t.Errorf("confirmed = %#v, want [%q]", evs[0].Data["confirmed"], releaseupdate.ComponentDaemon)
}
if !strings.Contains(evs[0].Detail, "confirmed running") {
t.Errorf("Detail = %q, want it to say confirmed running", evs[0].Detail)
}
if _, found, err := releaseupdate.ReadPendingReport(path); err != nil || found {
t.Errorf("report still present after reportPendingUpdate: found=%v err=%v", found, err)
}
}
// TestReportPendingUpdateClearsUnreadableReport mirrors the telegram
// behaviour: a corrupt report must not jam every future boot.
func TestReportPendingUpdateClearsUnreadableReport(t *testing.T) {
s := newTestServer(t)
path := filepath.Join(t.TempDir(), "update-report.json")
if err := os.WriteFile(path, []byte("{not valid json"), 0o600); err != nil {
t.Fatal(err)
}
s.UpdateReportPath = path
s.reportPendingUpdate(context.Background())
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("corrupt report file still exists after reportPendingUpdate: err = %v", err)
}
}
func TestReportPendingUpdateNoFileIsNoop(t *testing.T) {
s := newTestServer(t)
s.UpdateReportPath = filepath.Join(t.TempDir(), "does-not-exist.json")
s.reportPendingUpdate(context.Background())
evs, err := s.Store.EventsSince(context.Background(), 0, 10)
if err != nil {
t.Fatal(err)
}
if len(evs) != 0 {
t.Fatalf("events = %#v, want none", evs)
}
}
func TestReportPendingUpdateDisabledWhenPathEmpty(t *testing.T) {
s := newTestServer(t)
s.reportPendingUpdate(context.Background())
evs, err := s.Store.EventsSince(context.Background(), 0, 10)
if err != nil {
t.Fatal(err)
}
if len(evs) != 0 {
t.Fatalf("events = %#v, want none", evs)
}
}
// TestSummarizeUpdateReportDistinguishesDriftFromConfirmed is the false-
// success regression the whole verification mechanism (c25799e,
// archie-core-3jag's sibling) exists to catch: a passed health probe alone
// must never be worded as a confirmed version.
func TestSummarizeUpdateReportDistinguishesDriftFromConfirmed(t *testing.T) {
report := releaseupdate.Report{
HealthCheck: "passed",
Previous: map[string]string{releaseupdate.ComponentDaemon: "1.0.0"},
Installed: map[string]string{releaseupdate.ComponentDaemon: "1.1.0"},
}
running := map[string]string{releaseupdate.ComponentDaemon: "1.0.0"}
verification := report.Verify(running)
got := summarizeUpdateReport(report, verification)
if !strings.Contains(got, "did NOT take effect") {
t.Errorf("summary = %q, want it to flag drift rather than claim success", got)
}
if strings.Contains(got, "confirmed running") {
t.Errorf("summary = %q, must not claim confirmed running when drift was detected", got)
}
}
// TestHandleChatUpdateInstallSetsReportPath is the regression proof: before
// this fix, the webui install handler passed
// releaseupdate.InstallMeta{Channel: "webui"} with no ReportPath, so the