diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 9d874ab16..afe4faf48 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -366,7 +366,7 @@ func (b *boot) loadCatalog(ctx context.Context, cfgPath string) { // setupObservability builds the event bus, channel manager and dashboard // server. Every event is logged to SQLite (stamped with its row id) and // then fanned out to live dashboard connections. -func (b *boot) setupObservability() { +func (b *boot) setupObservability(ctx context.Context) { cfg, log := b.cfg, b.log bus := events.NewBus() b.bus = bus @@ -388,6 +388,10 @@ func (b *boot) setupObservability() { } b.web = &webui.Server{Store: b.stateStore, Log: log.With("component", "webui"), LogFeed: b.logFeed, TaskLogs: b.taskLogs, Cfg: config.NewHolder(cfg), Channels: b.channelManager, Events: bus} b.web.UpdateReportPath = updateReportPath(cfg.WorkDir, "webui") + // The watchdog leaves its verdict in a file on this host, so the daemon + // reads it and publishes the outcome as an event; the dashboard renders + // what it receives, wherever it runs (archie-core-8cda.5.4). + b.startUpdateRelay(ctx, b.web.UpdateReportPath) if (cfg.Chat.Telegram.Token != (secret.SecretRef{}) || cfg.Chat.Telegram.TokenEnv != "") && len(cfg.Chat.Telegram.AllowedUserIDs) > 0 { b.web.TelegramUpdateReportPath = updateReportPath(cfg.WorkDir, cfg.BotUser) b.web.TelegramUpdateChatID = cfg.Chat.Telegram.AllowedUserIDs[0] @@ -403,7 +407,7 @@ func (b *boot) setupObservability() { } b.wireWebStoreSurfaces() sink := bus.Subscribe(256) - go persistAndBroadcastEvents(sink, b.stateStore, b.web, log) + go persistAndBroadcastEvents(ctx, sink, b.stateStore, b.web, log) } // wireWebStoreSurfaces attaches the dashboard's optional storage surfaces. diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 9ed6903fb..f16f88cdf 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -399,7 +399,7 @@ func Run() int { //nolint:cyclop // the composition root's setup sequence is del return 0 } b.loadCatalog(ctx, args.cfgPath) - b.setupObservability() + b.setupObservability(ctx) if err := b.setupBackends(ctx); err != nil { return 1 @@ -509,10 +509,17 @@ func recordBootOverlayError(p *atomic.Pointer[string], log *slog.Logger, err err log.Error(msg, args...) } -func persistAndBroadcastEvents(sink *events.Sub, st store.TaskStore, web *webui.Server, log *slog.Logger) { +// persistAndBroadcastEvents drains the bus until it closes, giving each +// event an ID from the store before fanning it out. +// +// The insert deliberately outlives ctx: this loop ends when the bus closes, +// which is part of shutdown, and the last events of a run are exactly the +// ones an operator wants to read afterwards. +func persistAndBroadcastEvents(ctx context.Context, sink *events.Sub, st store.TaskStore, web *webui.Server, log *slog.Logger) { + ctx = context.WithoutCancel(ctx) for e := range sink.C { if e.ID == 0 { - id, err := st.InsertEvent(context.Background(), e) + id, err := st.InsertEvent(ctx, e) if err != nil { log.Error("event sink insert failed", "err", err) continue diff --git a/internal/app/archied/state_store_consumers_test.go b/internal/app/archied/state_store_consumers_test.go index e2967dcb0..08e587968 100644 --- a/internal/app/archied/state_store_consumers_test.go +++ b/internal/app/archied/state_store_consumers_test.go @@ -93,7 +93,7 @@ func TestSetupObservabilityRoutesTaskStoreThroughStateStore(t *testing.T) { taskLogs: logging.NewTaskRegistry(filepath.Join(t.TempDir(), "logs"), logFeed, logging.TaskSinkOptions{}), } t.Cleanup(b.cleanup) - b.setupObservability() + b.setupObservability(t.Context()) if b.web.Store != storeB { t.Fatalf("webui Store = %p, want b.stateStore (%p); dashboard task store did not route through the State Store adapter", b.web.Store, storeB) diff --git a/internal/app/archied/update_relay.go b/internal/app/archied/update_relay.go new file mode 100644 index 000000000..9d26ed348 --- /dev/null +++ b/internal/app/archied/update_relay.go @@ -0,0 +1,63 @@ +package archied + +import ( + "context" + "time" + + "github.com/samcharles93/archie-core/internal/releaseupdate" +) + +// updateReportPollInterval closes the watchdog/startup race: the watchdog +// writes its result only once this process answers /healthz, so a read at +// boot alone can always miss the report describing that very boot. +const updateReportPollInterval = 2 * time.Second + +// startUpdateRelay publishes the update watchdog's phase-2 outcome onto the +// activity stream. +// +// The daemon owns this because the daemon owns the update: the watchdog +// restarts archied and leaves its verdict in a file on archied's host, so +// only archied can read it. It previously ran inside the dashboard's own +// Run, which put a host-local file read in the process that is moving off +// the host (archie-core-8cda.5.4). The report reaches an operator the same +// way it always did, as an event -- persisted with every other event and +// delivered to whichever dashboard is watching. +// +// A report is cleared as soon as it is handled, relayed or unreadable, so a +// later boot never retries it and two boots never race to relay one outcome. +func (b *boot) startUpdateRelay(ctx context.Context, path string) { + if path == "" || b.bus == nil { + return + } + b.relayUpdateReport(path) + go func() { + ticker := time.NewTicker(updateReportPollInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + b.relayUpdateReport(path) + } + } + }() +} + +func (b *boot) relayUpdateReport(path string) { + report, found, err := releaseupdate.ReadPendingReport(path) + if err != nil { + b.log.Warn("read pending update report failed; discarding it", "err", err) + if clearErr := releaseupdate.ClearPendingReport(path); clearErr != nil { + b.log.Warn("clear unreadable update report failed", "err", clearErr) + } + return + } + if !found { + return + } + b.publishEvent(releaseupdate.ReportEvent(report, daemonRunningVersions(b.agentStatus))) + if err := releaseupdate.ClearPendingReport(path); err != nil { + b.log.Warn("clear pending update report failed", "err", err) + } +} diff --git a/internal/app/archied/update_relay_test.go b/internal/app/archied/update_relay_test.go new file mode 100644 index 000000000..3bb825a32 --- /dev/null +++ b/internal/app/archied/update_relay_test.go @@ -0,0 +1,121 @@ +package archied + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/releaseupdate" +) + +// relayBoot is the daemon reduced to what the relay needs: a bus to publish +// on and a logger. +func relayBoot(t *testing.T) (*boot, *events.Sub) { + t.Helper() + bus := events.NewBus() + t.Cleanup(bus.Close) + sub := bus.Subscribe(4) + return &boot{bus: bus, log: slog.New(slog.DiscardHandler)}, sub +} + +func waitForEvent(t *testing.T, sub *events.Sub) (events.Event, bool) { + t.Helper() + select { + case e := <-sub.C: + return e, true + case <-time.After(2 * time.Second): + return events.Event{}, false + } +} + +// TestRelayUpdateReportPublishesAndClears: the watchdog leaves its verdict in +// a file on the daemon's host, so the daemon reads it and puts the outcome on +// the activity stream. Relaying twice would tell the operator the same update +// finished twice, so the report is cleared as soon as it is handled +// (archie-core-nln7, rehomed by archie-core-8cda.5.4). +func TestRelayUpdateReportPublishesAndClears(t *testing.T) { + b, sub := relayBoot(t) + path := filepath.Join(t.TempDir(), "update-report.json") + report := releaseupdate.Report{ + Channel: "webui", + Previous: map[string]string{releaseupdate.ComponentDaemon: "1.0.0"}, + Installed: map[string]string{releaseupdate.ComponentDaemon: "1.1.0"}, + HealthCheck: "passed", + } + if err := releaseupdate.WritePendingReport(path, report); err != nil { + t.Fatal(err) + } + + b.relayUpdateReport(path) + + e, ok := waitForEvent(t, sub) + if !ok { + t.Fatal("no event published for a pending update report") + } + if e.Kind != events.KindUpdateReport { + t.Errorf("kind = %q, want %q", e.Kind, events.KindUpdateReport) + } + if e.Data["health_check"] != "passed" { + t.Errorf("health_check = %v, want passed", e.Data["health_check"]) + } + if _, found, err := releaseupdate.ReadPendingReport(path); err != nil || found { + t.Errorf("report still present after relaying: found=%v err=%v", found, err) + } + + // A second pass has nothing left to relay. + b.relayUpdateReport(path) + if e, ok := waitForEventQuickly(sub); ok { + t.Fatalf("a cleared report was relayed again: %+v", e) + } +} + +func waitForEventQuickly(sub *events.Sub) (events.Event, bool) { + select { + case e := <-sub.C: + return e, true + case <-time.After(100 * time.Millisecond): + return events.Event{}, false + } +} + +// TestRelayUpdateReportClearsUnreadableReport: a corrupt report must not jam +// every future boot behind it. +func TestRelayUpdateReportClearsUnreadableReport(t *testing.T) { + b, _ := relayBoot(t) + path := filepath.Join(t.TempDir(), "update-report.json") + if err := os.WriteFile(path, []byte("{not valid json"), 0o600); err != nil { + t.Fatal(err) + } + + b.relayUpdateReport(path) + + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("corrupt report file still exists after relaying: err = %v", err) + } +} + +func TestRelayUpdateReportWithNoFileIsSilent(t *testing.T) { + b, sub := relayBoot(t) + + b.relayUpdateReport(filepath.Join(t.TempDir(), "does-not-exist.json")) + + if e, ok := waitForEventQuickly(sub); ok { + t.Fatalf("published %+v with no report to relay", e) + } +} + +// TestStartUpdateRelayWithoutAPathDoesNothing: a deployment with no phase-2 +// reporting configured must not start a poller for a file that will never +// exist. +func TestStartUpdateRelayWithoutAPathDoesNothing(t *testing.T) { + b, sub := relayBoot(t) + + b.startUpdateRelay(t.Context(), "") + + if e, ok := waitForEventQuickly(sub); ok { + t.Fatalf("published %+v with no report path configured", e) + } +} diff --git a/internal/webui/update_report.go b/internal/releaseupdate/report_event.go similarity index 54% rename from internal/webui/update_report.go rename to internal/releaseupdate/report_event.go index 0a13744d4..aa4d5e33f 100644 --- a/internal/webui/update_report.go +++ b/internal/releaseupdate/report_event.go @@ -1,66 +1,20 @@ -package webui +package releaseupdate import ( - "context" "fmt" "sort" "strings" "github.com/samcharles93/archie-core/internal/events" - "github.com/samcharles93/archie-core/internal/releaseupdate" ) -// reportPendingUpdate checks for a phase-2 outcome left by the update -// watchdog for a dashboard-initiated install and relays it onto the -// activity stream, mirroring internal/channels/telegram/update.go's -// reportPendingUpdate for the Telegram channel. Called once on every daemon -// boot (Server.Run), so it fires whether this is the restart the update -// itself triggered or a later, unrelated restart. -// -// A report is cleared as soon as it's handled -- relayed, or found -// unreadable -- so a later boot never retries it and two boots never race -// to relay the same outcome. -func (s *Server) reportPendingUpdate(ctx context.Context) { - s.updateReportMu.Lock() - defer s.updateReportMu.Unlock() - if s.UpdateReportPath == "" { - return - } - report, found, err := releaseupdate.ReadPendingReport(s.UpdateReportPath) - if err != nil { - s.logf("read pending update report failed; discarding it", "err", err) - if clearErr := releaseupdate.ClearPendingReport(s.UpdateReportPath); clearErr != nil { - s.logf("clear unreadable update report failed", "err", clearErr) - } - return - } - if !found { - return - } - s.emit(ctx, updateReportEvent(report, s.runningVersions())) - if err := releaseupdate.ClearPendingReport(s.UpdateReportPath); err != nil { - s.logf("clear pending update report failed", "err", err) - } -} - -// runningVersions returns the versions components report for themselves, or -// nil when the composition root wired none -- see the identical seam on -// internal/channels/telegram.Gateway. Nil is not a no-op: with nothing to -// check against, every claim comes back Unverified rather than Confirmed. -func (s *Server) runningVersions() map[string]string { - if s.RunningVersions == nil { - return nil - } - return s.RunningVersions() -} - -// updateReportEvent turns a phase-2 Report, plus its verification against +// ReportEvent turns a phase-2 Report, plus its verification against // the versions actually running, into the activity-stream event the // dashboard renders. Unlike Telegram's formatPendingReport, this ships the // structured Verification rather than pre-rendered prose, so the frontend // controls presentation -- the two channels' operators read the same facts // through different UIs, not necessarily the same sentence. -func updateReportEvent(report releaseupdate.Report, running map[string]string) events.Event { +func ReportEvent(report Report, running map[string]string) events.Event { verification := report.Verify(running) drift := make([]map[string]any, 0, len(verification.Drift)) for _, d := range verification.Drift { @@ -73,7 +27,7 @@ func updateReportEvent(report releaseupdate.Report, running map[string]string) e } return events.Event{ Kind: events.KindUpdateReport, - Detail: summarizeUpdateReport(report, verification), + Detail: summarizeReport(report, verification), Data: map[string]any{ "health_check": report.HealthCheck, "rolled_back": report.RolledBack, @@ -86,14 +40,14 @@ func updateReportEvent(report releaseupdate.Report, running map[string]string) e } } -// summarizeUpdateReport is the one-line summary shown in the dashboard's +// summarizeReport is the one-line summary shown in the dashboard's // activity feed, which renders event.Detail generically for every event // kind (see ui/src/dashboard/dashboard.js activityRow). It condenses the // same facts internal/channels/telegram/update.go's formatPendingReport // spells out at length -- not a byte-identical wording, since the two // surfaces render differently, but the same four outcomes: confirmed, // drifted, unverified, or rolled back. -func summarizeUpdateReport(report releaseupdate.Report, verification releaseupdate.Verification) string { +func summarizeReport(report Report, verification Verification) string { if report.RolledBack { return "Update failed its health check and was rolled back to the previous version." } diff --git a/internal/releaseupdate/report_event_test.go b/internal/releaseupdate/report_event_test.go new file mode 100644 index 000000000..3afadf8f2 --- /dev/null +++ b/internal/releaseupdate/report_event_test.go @@ -0,0 +1,57 @@ +package releaseupdate + +import ( + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/events" +) + +// TestReportEventDistinguishesDriftFromConfirmed is the false-success +// regression the whole verification mechanism exists to catch: a passed +// health probe alone must never be worded as a confirmed version. +func TestReportEventDistinguishesDriftFromConfirmed(t *testing.T) { + report := Report{ + HealthCheck: "passed", + Previous: map[string]string{ComponentDaemon: "1.0.0"}, + Installed: map[string]string{ComponentDaemon: "1.1.0"}, + } + + drifted := ReportEvent(report, map[string]string{ComponentDaemon: "1.0.0"}) + if drifted.Kind != events.KindUpdateReport { + t.Fatalf("kind = %q, want %q", drifted.Kind, events.KindUpdateReport) + } + if !strings.Contains(drifted.Detail, "did NOT take effect") { + t.Errorf("summary = %q, want it to flag drift rather than claim success", drifted.Detail) + } + if strings.Contains(drifted.Detail, "confirmed running") { + t.Errorf("summary = %q, must not claim confirmed running when drift was detected", drifted.Detail) + } + + confirmed := ReportEvent(report, map[string]string{ComponentDaemon: "1.1.0"}) + if !strings.Contains(confirmed.Detail, "confirmed running") { + t.Errorf("summary = %q, want it to confirm the running version", confirmed.Detail) + } + ids, _ := confirmed.Data["confirmed"].([]string) + if len(ids) != 1 || ids[0] != ComponentDaemon { + t.Errorf("confirmed = %#v, want [%q]", confirmed.Data["confirmed"], ComponentDaemon) + } +} + +// TestReportEventRolledBackIsNeverWordedAsSuccess. +func TestReportEventRolledBackIsNeverWordedAsSuccess(t *testing.T) { + report := Report{ + HealthCheck: "failed", + RolledBack: true, + Previous: map[string]string{ComponentDaemon: "1.0.0"}, + Installed: map[string]string{ComponentDaemon: "1.1.0"}, + } + + e := ReportEvent(report, map[string]string{ComponentDaemon: "1.0.0"}) + if !strings.Contains(e.Detail, "rolled back") { + t.Errorf("summary = %q, want it to report the rollback", e.Detail) + } + if e.Data["rolled_back"] != true { + t.Errorf("rolled_back = %v, want true", e.Data["rolled_back"]) + } +} diff --git a/internal/webui/server.go b/internal/webui/server.go index 743a1814f..f52043fb4 100644 --- a/internal/webui/server.go +++ b/internal/webui/server.go @@ -213,9 +213,8 @@ type Server struct { // When Cfg is present, Cfg.Get().Web.TrustForwardedHeaders is also checked. TrustForwardedHeaders bool - mu sync.Mutex - conns map[chan events.Event]struct{} - updateReportMu sync.Mutex + mu sync.Mutex + conns map[chan events.Event]struct{} } func (s *Server) trustForwardedHeaders() bool { @@ -390,8 +389,6 @@ func trimLeadingSlash(p string) string { // Run serves until ctx ends. func (s *Server) Run(ctx context.Context, listen string) error { - s.reportPendingUpdate(ctx) - go s.watchPendingUpdate(ctx) srv := &http.Server{Addr: listen, Handler: s.Handler(), ReadHeaderTimeout: 5 * time.Second} go func() { <-ctx.Done() @@ -406,22 +403,6 @@ func (s *Server) Run(ctx context.Context, listen string) error { return nil } -// watchPendingUpdate closes the watchdog/startup race: the watchdog writes -// the health result only after this process answers /healthz, so a startup -// read alone can always miss a report for the very boot it describes. -func (s *Server) watchPendingUpdate(ctx context.Context) { - ticker := time.NewTicker(2 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - s.reportPendingUpdate(ctx) - } - } -} - func writeJSON(w http.ResponseWriter, v any) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(v) diff --git a/internal/webui/update_report_test.go b/internal/webui/update_report_test.go index d341bbeeb..2048b0e89 100644 --- a/internal/webui/update_report_test.go +++ b/internal/webui/update_report_test.go @@ -6,131 +6,13 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "os" "path/filepath" - "strings" "testing" - "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/gateway" "github.com/samcharles93/archie-core/internal/releaseupdate" ) -// TestReportPendingUpdateEmitsAndClearsReport is the webui counterpart of -// telegram's TestReportPendingUpdateSendsAndClearsReport: a dashboard- -// initiated install must produce a phase-2 report that reaches the -// dashboard, and must never be relayed twice (archie-core-nln7). -func TestReportPendingUpdateEmitsAndClearsReport(t *testing.T) { - s := newTestServer(t) - path := filepath.Join(t.TempDir(), "update-report.json") - report := releaseupdate.Report{ - Channel: "webui", - Previous: map[string]string{releaseupdate.ComponentDaemon: "1.0.0"}, - Installed: map[string]string{releaseupdate.ComponentDaemon: "1.1.0"}, - HealthCheck: "passed", - } - if err := releaseupdate.WritePendingReport(path, report); err != nil { - t.Fatal(err) - } - s.UpdateReportPath = path - s.RunningVersions = func() map[string]string { - return map[string]string{releaseupdate.ComponentDaemon: "1.1.0"} - } - - ctx := context.Background() - s.reportPendingUpdate(ctx) - - evs, err := s.Store.EventsSince(ctx, 0, 10) - if err != nil { - t.Fatal(err) - } - if len(evs) != 1 { - t.Fatalf("events = %#v, want exactly one", evs) - } - if evs[0].Kind != events.KindUpdateReport { - t.Errorf("kind = %q, want %q", evs[0].Kind, events.KindUpdateReport) - } - confirmed, _ := evs[0].Data["confirmed"].([]any) - if len(confirmed) != 1 || confirmed[0] != releaseupdate.ComponentDaemon { - t.Errorf("confirmed = %#v, want [%q]", evs[0].Data["confirmed"], releaseupdate.ComponentDaemon) - } - if !strings.Contains(evs[0].Detail, "confirmed running") { - t.Errorf("Detail = %q, want it to say confirmed running", evs[0].Detail) - } - - if _, found, err := releaseupdate.ReadPendingReport(path); err != nil || found { - t.Errorf("report still present after reportPendingUpdate: found=%v err=%v", found, err) - } -} - -// TestReportPendingUpdateClearsUnreadableReport mirrors the telegram -// behaviour: a corrupt report must not jam every future boot. -func TestReportPendingUpdateClearsUnreadableReport(t *testing.T) { - s := newTestServer(t) - path := filepath.Join(t.TempDir(), "update-report.json") - if err := os.WriteFile(path, []byte("{not valid json"), 0o600); err != nil { - t.Fatal(err) - } - s.UpdateReportPath = path - - s.reportPendingUpdate(context.Background()) - - if _, err := os.Stat(path); !os.IsNotExist(err) { - t.Fatalf("corrupt report file still exists after reportPendingUpdate: err = %v", err) - } -} - -func TestReportPendingUpdateNoFileIsNoop(t *testing.T) { - s := newTestServer(t) - s.UpdateReportPath = filepath.Join(t.TempDir(), "does-not-exist.json") - - s.reportPendingUpdate(context.Background()) - - evs, err := s.Store.EventsSince(context.Background(), 0, 10) - if err != nil { - t.Fatal(err) - } - if len(evs) != 0 { - t.Fatalf("events = %#v, want none", evs) - } -} - -func TestReportPendingUpdateDisabledWhenPathEmpty(t *testing.T) { - s := newTestServer(t) - - s.reportPendingUpdate(context.Background()) - - evs, err := s.Store.EventsSince(context.Background(), 0, 10) - if err != nil { - t.Fatal(err) - } - if len(evs) != 0 { - t.Fatalf("events = %#v, want none", evs) - } -} - -// TestSummarizeUpdateReportDistinguishesDriftFromConfirmed is the false- -// success regression the whole verification mechanism (c25799e, -// archie-core-3jag's sibling) exists to catch: a passed health probe alone -// must never be worded as a confirmed version. -func TestSummarizeUpdateReportDistinguishesDriftFromConfirmed(t *testing.T) { - report := releaseupdate.Report{ - HealthCheck: "passed", - Previous: map[string]string{releaseupdate.ComponentDaemon: "1.0.0"}, - Installed: map[string]string{releaseupdate.ComponentDaemon: "1.1.0"}, - } - running := map[string]string{releaseupdate.ComponentDaemon: "1.0.0"} - verification := report.Verify(running) - - got := summarizeUpdateReport(report, verification) - if !strings.Contains(got, "did NOT take effect") { - t.Errorf("summary = %q, want it to flag drift rather than claim success", got) - } - if strings.Contains(got, "confirmed running") { - t.Errorf("summary = %q, must not claim confirmed running when drift was detected", got) - } -} - // TestHandleChatUpdateInstallSetsReportPath is the regression proof: before // this fix, the webui install handler passed // releaseupdate.InstallMeta{Channel: "webui"} with no ReportPath, so the