From e6791746ae69fe265448ac0e7164f968fda26560 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Thu, 8 Oct 2026 22:36:09 +1100 Subject: [PATCH] fix(archiemessaging): recover a channel that fails on its own --- internal/app/archiemessaging/compose.go | 12 +- internal/app/archiemessaging/supervise.go | 150 +++++++++++++++-- .../app/archiemessaging/supervise_test.go | 156 ++++++++++++++++++ 3 files changed, 300 insertions(+), 18 deletions(-) create mode 100644 internal/app/archiemessaging/supervise_test.go diff --git a/internal/app/archiemessaging/compose.go b/internal/app/archiemessaging/compose.go index f639ef16b..4d6faf77d 100644 --- a/internal/app/archiemessaging/compose.go +++ b/internal/app/archiemessaging/compose.go @@ -58,7 +58,7 @@ type deps struct { } // channelInstance is one composed channel and what is needed to restart it. -// mu guards channel and cancel. +// mu guards every field below. type channelInstance struct { name string rebuild func(ResolvedConfig) (channels.Channel, error) @@ -68,6 +68,9 @@ type channelInstance struct { cancel context.CancelFunc supervised bool restartPending bool + // notify wakes a supervisor that is between runs, so an explicit restart + // starts the replacement instead of waiting out a backoff or a park. + notify chan struct{} } // current returns the channel instance currently serving, under the lock. @@ -108,6 +111,9 @@ type Service struct { appliedVersion atomic.Int64 // reconcileInterval is how often the stored settings are re-read. reconcileInterval time.Duration + // retry bounds how a supervisor retries a channel that fails on its own + // before parking it for an explicit restart. + retry channelRetryPolicy mu sync.Mutex running bool @@ -135,6 +141,7 @@ func compose(ctx context.Context, d deps) (*Service, error) { if srv.reconcileInterval <= 0 { srv.reconcileInterval = applystatus.RestampInterval } + srv.retry = defaultChannelRetry() instances, err := composeChannels(ctx, d) if err != nil { @@ -167,6 +174,9 @@ func composeChannels(ctx context.Context, d deps) ([]*channelInstance, error) { } instances = append(instances, instance) } + for _, instance := range instances { + instance.notify = make(chan struct{}, 1) + } return instances, nil } diff --git a/internal/app/archiemessaging/supervise.go b/internal/app/archiemessaging/supervise.go index 222608c82..ce9500d84 100644 --- a/internal/app/archiemessaging/supervise.go +++ b/internal/app/archiemessaging/supervise.go @@ -4,14 +4,21 @@ import ( "context" "errors" "fmt" + "time" "github.com/samcharles93/archie-core/internal/channels" ) -// runChannel runs one channel and starts its replacement after a requested -// restart. A channel that exits on its own is not restarted. +// runChannel supervises one channel for the service's lifetime. It starts the +// channel and, when the run ends, decides what happens next: a requested +// restart starts the replacement, a run that failed is retried with bounded +// backoff, and a channel that keeps failing is parked until an explicit +// restart. Only shutdown ends the supervisor, so a channel that stops is never +// left without one to start it again. func (s *Service) runChannel(ctx context.Context, c *channelInstance) { defer s.wg.Done() + defer c.supervisorExits() + attempts := 0 for { c.mu.Lock() ch := c.channel @@ -24,7 +31,7 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) { // Decide the next state under the same lock a restart takes, so a // restart racing this exit is either seen here (restartPending) or - // observed as a channel that is no longer supervised. + // delivered to the wait below (notify). c.mu.Lock() c.cancel = nil shutdown := ctx.Err() != nil @@ -32,9 +39,6 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) { if !shutdown && restart { c.restartPending = false } - if shutdown || !restart { - c.supervised = false - } c.mu.Unlock() cancel() @@ -52,23 +56,122 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) { return } if restart { - // The replacement is already in c.channel (see restartChannel); the - // next pass starts it. + attempts = 0 continue } - // The channel stopped on its own. - if err != nil { - s.status.MarkFailed(c.name, err.Error()) - s.log.Error("channel stopped with error", "name", c.name, "err", err) - } else { + if err == nil { + // The channel stopped itself. Hold it stopped rather than + // spending the failure retry budget on a run that chose to end. s.status.MarkStopped(c.name, "") + s.log.Info("channel stopped", "name", c.name) + if s.awaitChannel(ctx, c, 0) == channelShutdown { + return + } + attempts = 0 + continue + } + attempts++ + s.status.MarkFailed(c.name, err.Error()) + if attempts >= s.retry.attempts { + s.log.Error("channel failed repeatedly; parked until an explicit restart", + "name", c.name, "attempts", attempts, "err", err) + if s.awaitChannel(ctx, c, 0) == channelShutdown { + return + } + attempts = 0 + continue + } + backoff := s.retry.backoffFor(attempts) + s.log.Warn("channel failed; restarting", "name", c.name, + "attempt", attempts, "of", s.retry.attempts, "in", backoff, "err", err) + switch s.awaitChannel(ctx, c, backoff) { + case channelShutdown: + return + case channelRestart: + attempts = 0 } - return } } -// restartChannel builds a replacement for channel id from next, then stops -// the old one and starts it. A channel not running is not restarted. +// channelRetry bounds how a supervisor retries a channel that fails on its own +// before parking it for an explicit restart. +type channelRetryPolicy struct { + attempts int + backoff time.Duration + maxBackoff time.Duration +} + +// defaultChannelRetry is the retry policy a composed service uses. +func defaultChannelRetry() channelRetryPolicy { + return channelRetryPolicy{attempts: 5, backoff: time.Second, maxBackoff: 30 * time.Second} +} + +// backoffFor is the wait before attempt n (1-based): doubling from backoff, +// capped at maxBackoff so a long outage does not become a long silence between +// retries. +func (r channelRetryPolicy) backoffFor(attempt int) time.Duration { + backoff := r.backoff << (attempt - 1) + if backoff <= 0 || backoff > r.maxBackoff { + return r.maxBackoff + } + return backoff +} + +// channelWait is how a supervisor's wait between runs ended. +type channelWait int + +const ( + // channelRetry: the backoff elapsed; run the channel again. + channelRetry channelWait = iota + // channelRestart: an explicit restart arrived; start the replacement. + channelRestart + // channelShutdown: the service is stopping. + channelShutdown +) + +// awaitChannel blocks until an explicit restart arrives, the service stops, or +// the backoff elapses. A delay of zero waits with no timer: the parked state +// after a channel stopped or gave up, which only a restart or shutdown ends. +func (s *Service) awaitChannel(ctx context.Context, c *channelInstance, delay time.Duration) channelWait { + if delay <= 0 { + select { + case <-ctx.Done(): + return channelShutdown + case <-c.notify: + return channelRestart + } + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return channelShutdown + case <-c.notify: + return channelRestart + case <-timer.C: + // A restart delivered as the timer fired is read here rather than + // lost between the two branches. + select { + case <-c.notify: + return channelRestart + default: + return channelRetry + } + } +} + +// supervisorExits records that a channel's supervisor is leaving, so a restart +// after shutdown is refused rather than signalling a goroutine that is gone. +func (c *channelInstance) supervisorExits() { + c.mu.Lock() + c.supervised = false + c.mu.Unlock() +} + +// restartChannel builds a replacement for channel id from next and starts it in +// place of the running one. A supervisor that is between runs -- backing off a +// failure, or parked after giving up -- is woken to start the replacement; a +// channel with no supervisor (never started, or shut down) is refused. func (s *Service) restartChannel(id string, next ResolvedConfig) error { c := s.instanceByID(id) if c == nil { @@ -90,11 +193,24 @@ func (s *Service) restartChannel(id string, next ResolvedConfig) error { return fmt.Errorf("channel %q is not running", id) } c.channel = replacement - c.restartPending = true cancel := c.cancel + if cancel != nil { + // A run is in flight. Cancelling its context ends Start, and the + // supervisor sees restartPending and starts the replacement. + c.restartPending = true + } else { + // The supervisor is between runs. Wake it so it starts the replacement + // now, rather than waiting out its backoff or its park. + c.restartPending = false + } c.mu.Unlock() if cancel != nil { cancel() + return nil + } + select { + case c.notify <- struct{}{}: + default: } return nil } diff --git a/internal/app/archiemessaging/supervise_test.go b/internal/app/archiemessaging/supervise_test.go new file mode 100644 index 000000000..ae36788ba --- /dev/null +++ b/internal/app/archiemessaging/supervise_test.go @@ -0,0 +1,156 @@ +package archiemessaging + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "sync" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/channels" + "github.com/samcharles93/archie-core/internal/domain/messaging" +) + +// fakeChannel is a channel whose runs a test scripts: it counts starts and +// either fails immediately or blocks until its run context ends. +type fakeChannel struct { + mu sync.Mutex + starts int + runErr error +} + +func (f *fakeChannel) Name() string { return "fake" } + +func (f *fakeChannel) Start(ctx context.Context, _ messaging.ChatContract, _ channels.Lifecycle) error { + f.mu.Lock() + f.starts++ + f.mu.Unlock() + if f.runErr != nil { + return f.runErr + } + <-ctx.Done() + return nil +} + +func (f *fakeChannel) Stop(context.Context) error { return nil } + +func (f *fakeChannel) ConfigSchema() json.RawMessage { return nil } + +func (f *fakeChannel) ValidateConfig(map[string]any) error { return nil } + +func (f *fakeChannel) count() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.starts +} + +// startService composes and starts a service over one scripted channel, with a +// retry policy the test drives without waiting out real backoff. +func startService(t *testing.T, ch *fakeChannel, retry channelRetryPolicy) *Service { + t.Helper() + inst := &channelInstance{ + name: "fake", + channel: ch, + rebuild: func(ResolvedConfig) (channels.Channel, error) { return ch, nil }, + } + srv, err := compose(context.Background(), deps{Log: slog.Default(), ExtensionChannels: []*channelInstance{inst}}) + if err != nil { + t.Fatalf("compose: %v", err) + } + srv.retry = retry + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = srv.Start(ctx) + }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("service did not stop after cancel") + } + }) + return srv +} + +func waitStarts(t *testing.T, ch *fakeChannel, want int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if ch.count() >= want { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("channel started %d times, want at least %d", ch.count(), want) +} + +// TestChannelSupervisorRetriesAndRestarts pins that a channel that fails on its +// own is retried with bounded backoff and, once the budget is spent, parked so +// an explicit restart brings it back rather than being refused. +func TestChannelSupervisorRetriesAndRestarts(t *testing.T) { + ch := &fakeChannel{runErr: errors.New("transport down")} + srv := startService(t, ch, channelRetryPolicy{attempts: 3, backoff: time.Millisecond, maxBackoff: time.Millisecond}) + + // The supervisor retries the failed run instead of abandoning the channel. + waitStarts(t, ch, 3) + + if err := srv.restartChannel("fake", ResolvedConfig{}); err != nil { + t.Fatalf("restart after the retry budget was spent: %v", err) + } + waitStarts(t, ch, 4) +} + +// TestChannelSupervisorRestartReplacesRunningChannel pins the in-flight path: a +// restart of a channel whose run is live cancels the run and starts the +// replacement, with no backoff. +func TestChannelSupervisorRestartReplacesRunningChannel(t *testing.T) { + ch := &fakeChannel{} // blocks until its run context ends + srv := startService(t, ch, channelRetryPolicy{attempts: 3, backoff: time.Hour, maxBackoff: time.Hour}) + + waitStarts(t, ch, 1) + if err := srv.restartChannel("fake", ResolvedConfig{}); err != nil { + t.Fatalf("restart running channel: %v", err) + } + waitStarts(t, ch, 2) +} + +// TestChannelRestartRefusedWithoutSupervisor pins that a channel with no +// supervisor -- never started, or already shut down -- is refused rather than +// signalling a goroutine that is gone. +func TestChannelRestartRefusedWithoutSupervisor(t *testing.T) { + ch := &fakeChannel{} + inst := &channelInstance{ + name: "fake", + channel: ch, + rebuild: func(ResolvedConfig) (channels.Channel, error) { return ch, nil }, + } + srv, err := compose(context.Background(), deps{Log: slog.Default(), ExtensionChannels: []*channelInstance{inst}}) + if err != nil { + t.Fatalf("compose: %v", err) + } + if err := srv.restartChannel("fake", ResolvedConfig{}); err == nil { + t.Fatal("restart before the service started should be refused") + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = srv.Start(ctx) + }() + waitStarts(t, ch, 1) + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("service did not stop") + } + if err := srv.restartChannel("fake", ResolvedConfig{}); err == nil { + t.Fatal("restart after shutdown should be refused") + } +}