fix(archiemessaging): recover a channel that fails on its own

This commit is contained in:
sam committed 2026-10-08 22:36:09 +11:00
1 parent 4183fa4442
commit e6791746ae
3 files changed
+300 -18

No files matched your search

+11 -1
View File
@@ -58,7 +58,7 @@ type deps struct {
}
// channelInstance is one composed channel and what is needed to restart it.
// mu guards channel and cancel.
// mu guards every field below.
type channelInstance struct {
name string
rebuild func(ResolvedConfig) (channels.Channel, error)
@@ -68,6 +68,9 @@ type channelInstance struct {
cancel context.CancelFunc
supervised bool
restartPending bool
// notify wakes a supervisor that is between runs, so an explicit restart
// starts the replacement instead of waiting out a backoff or a park.
notify chan struct{}
}
// current returns the channel instance currently serving, under the lock.
@@ -108,6 +111,9 @@ type Service struct {
appliedVersion atomic.Int64
// reconcileInterval is how often the stored settings are re-read.
reconcileInterval time.Duration
// retry bounds how a supervisor retries a channel that fails on its own
// before parking it for an explicit restart.
retry channelRetryPolicy
mu sync.Mutex
running bool
@@ -135,6 +141,7 @@ func compose(ctx context.Context, d deps) (*Service, error) {
if srv.reconcileInterval <= 0 {
srv.reconcileInterval = applystatus.RestampInterval
}
srv.retry = defaultChannelRetry()
instances, err := composeChannels(ctx, d)
if err != nil {
@@ -167,6 +174,9 @@ func composeChannels(ctx context.Context, d deps) ([]*channelInstance, error) {
}
instances = append(instances, instance)
}
for _, instance := range instances {
instance.notify = make(chan struct{}, 1)
}
return instances, nil
}
+133 -17
View File
@@ -4,14 +4,21 @@ import (
"context"
"errors"
"fmt"
"time"
"github.com/samcharles93/archie-core/internal/channels"
)
// runChannel runs one channel and starts its replacement after a requested
// restart. A channel that exits on its own is not restarted.
// runChannel supervises one channel for the service's lifetime. It starts the
// channel and, when the run ends, decides what happens next: a requested
// restart starts the replacement, a run that failed is retried with bounded
// backoff, and a channel that keeps failing is parked until an explicit
// restart. Only shutdown ends the supervisor, so a channel that stops is never
// left without one to start it again.
func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
defer s.wg.Done()
defer c.supervisorExits()
attempts := 0
for {
c.mu.Lock()
ch := c.channel
@@ -24,7 +31,7 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
// Decide the next state under the same lock a restart takes, so a
// restart racing this exit is either seen here (restartPending) or
// observed as a channel that is no longer supervised.
// delivered to the wait below (notify).
c.mu.Lock()
c.cancel = nil
shutdown := ctx.Err() != nil
@@ -32,9 +39,6 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
if !shutdown && restart {
c.restartPending = false
}
if shutdown || !restart {
c.supervised = false
}
c.mu.Unlock()
cancel()
@@ -52,23 +56,122 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
return
}
if restart {
// The replacement is already in c.channel (see restartChannel); the
// next pass starts it.
attempts = 0
continue
}
// The channel stopped on its own.
if err != nil {
s.status.MarkFailed(c.name, err.Error())
s.log.Error("channel stopped with error", "name", c.name, "err", err)
} else {
if err == nil {
// The channel stopped itself. Hold it stopped rather than
// spending the failure retry budget on a run that chose to end.
s.status.MarkStopped(c.name, "")
s.log.Info("channel stopped", "name", c.name)
if s.awaitChannel(ctx, c, 0) == channelShutdown {
return
}
attempts = 0
continue
}
attempts++
s.status.MarkFailed(c.name, err.Error())
if attempts >= s.retry.attempts {
s.log.Error("channel failed repeatedly; parked until an explicit restart",
"name", c.name, "attempts", attempts, "err", err)
if s.awaitChannel(ctx, c, 0) == channelShutdown {
return
}
attempts = 0
continue
}
backoff := s.retry.backoffFor(attempts)
s.log.Warn("channel failed; restarting", "name", c.name,
"attempt", attempts, "of", s.retry.attempts, "in", backoff, "err", err)
switch s.awaitChannel(ctx, c, backoff) {
case channelShutdown:
return
case channelRestart:
attempts = 0
}
return
}
}
// restartChannel builds a replacement for channel id from next, then stops
// the old one and starts it. A channel not running is not restarted.
// channelRetry bounds how a supervisor retries a channel that fails on its own
// before parking it for an explicit restart.
type channelRetryPolicy struct {
attempts int
backoff time.Duration
maxBackoff time.Duration
}
// defaultChannelRetry is the retry policy a composed service uses.
func defaultChannelRetry() channelRetryPolicy {
return channelRetryPolicy{attempts: 5, backoff: time.Second, maxBackoff: 30 * time.Second}
}
// backoffFor is the wait before attempt n (1-based): doubling from backoff,
// capped at maxBackoff so a long outage does not become a long silence between
// retries.
func (r channelRetryPolicy) backoffFor(attempt int) time.Duration {
backoff := r.backoff << (attempt - 1)
if backoff <= 0 || backoff > r.maxBackoff {
return r.maxBackoff
}
return backoff
}
// channelWait is how a supervisor's wait between runs ended.
type channelWait int
const (
// channelRetry: the backoff elapsed; run the channel again.
channelRetry channelWait = iota
// channelRestart: an explicit restart arrived; start the replacement.
channelRestart
// channelShutdown: the service is stopping.
channelShutdown
)
// awaitChannel blocks until an explicit restart arrives, the service stops, or
// the backoff elapses. A delay of zero waits with no timer: the parked state
// after a channel stopped or gave up, which only a restart or shutdown ends.
func (s *Service) awaitChannel(ctx context.Context, c *channelInstance, delay time.Duration) channelWait {
if delay <= 0 {
select {
case <-ctx.Done():
return channelShutdown
case <-c.notify:
return channelRestart
}
}
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
return channelShutdown
case <-c.notify:
return channelRestart
case <-timer.C:
// A restart delivered as the timer fired is read here rather than
// lost between the two branches.
select {
case <-c.notify:
return channelRestart
default:
return channelRetry
}
}
}
// supervisorExits records that a channel's supervisor is leaving, so a restart
// after shutdown is refused rather than signalling a goroutine that is gone.
func (c *channelInstance) supervisorExits() {
c.mu.Lock()
c.supervised = false
c.mu.Unlock()
}
// restartChannel builds a replacement for channel id from next and starts it in
// place of the running one. A supervisor that is between runs -- backing off a
// failure, or parked after giving up -- is woken to start the replacement; a
// channel with no supervisor (never started, or shut down) is refused.
func (s *Service) restartChannel(id string, next ResolvedConfig) error {
c := s.instanceByID(id)
if c == nil {
@@ -90,11 +193,24 @@ func (s *Service) restartChannel(id string, next ResolvedConfig) error {
return fmt.Errorf("channel %q is not running", id)
}
c.channel = replacement
c.restartPending = true
cancel := c.cancel
if cancel != nil {
// A run is in flight. Cancelling its context ends Start, and the
// supervisor sees restartPending and starts the replacement.
c.restartPending = true
} else {
// The supervisor is between runs. Wake it so it starts the replacement
// now, rather than waiting out its backoff or its park.
c.restartPending = false
}
c.mu.Unlock()
if cancel != nil {
cancel()
return nil
}
select {
case c.notify <- struct{}{}:
default:
}
return nil
}
@@ -0,0 +1,156 @@
package archiemessaging
import (
"context"
"encoding/json"
"errors"
"log/slog"
"sync"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/channels"
"github.com/samcharles93/archie-core/internal/domain/messaging"
)
// fakeChannel is a channel whose runs a test scripts: it counts starts and
// either fails immediately or blocks until its run context ends.
type fakeChannel struct {
mu sync.Mutex
starts int
runErr error
}
func (f *fakeChannel) Name() string { return "fake" }
func (f *fakeChannel) Start(ctx context.Context, _ messaging.ChatContract, _ channels.Lifecycle) error {
f.mu.Lock()
f.starts++
f.mu.Unlock()
if f.runErr != nil {
return f.runErr
}
<-ctx.Done()
return nil
}
func (f *fakeChannel) Stop(context.Context) error { return nil }
func (f *fakeChannel) ConfigSchema() json.RawMessage { return nil }
func (f *fakeChannel) ValidateConfig(map[string]any) error { return nil }
func (f *fakeChannel) count() int {
f.mu.Lock()
defer f.mu.Unlock()
return f.starts
}
// startService composes and starts a service over one scripted channel, with a
// retry policy the test drives without waiting out real backoff.
func startService(t *testing.T, ch *fakeChannel, retry channelRetryPolicy) *Service {
t.Helper()
inst := &channelInstance{
name: "fake",
channel: ch,
rebuild: func(ResolvedConfig) (channels.Channel, error) { return ch, nil },
}
srv, err := compose(context.Background(), deps{Log: slog.Default(), ExtensionChannels: []*channelInstance{inst}})
if err != nil {
t.Fatalf("compose: %v", err)
}
srv.retry = retry
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
defer close(done)
_ = srv.Start(ctx)
}()
t.Cleanup(func() {
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Error("service did not stop after cancel")
}
})
return srv
}
func waitStarts(t *testing.T, ch *fakeChannel, want int) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if ch.count() >= want {
return
}
time.Sleep(time.Millisecond)
}
t.Fatalf("channel started %d times, want at least %d", ch.count(), want)
}
// TestChannelSupervisorRetriesAndRestarts pins that a channel that fails on its
// own is retried with bounded backoff and, once the budget is spent, parked so
// an explicit restart brings it back rather than being refused.
func TestChannelSupervisorRetriesAndRestarts(t *testing.T) {
ch := &fakeChannel{runErr: errors.New("transport down")}
srv := startService(t, ch, channelRetryPolicy{attempts: 3, backoff: time.Millisecond, maxBackoff: time.Millisecond})
// The supervisor retries the failed run instead of abandoning the channel.
waitStarts(t, ch, 3)
if err := srv.restartChannel("fake", ResolvedConfig{}); err != nil {
t.Fatalf("restart after the retry budget was spent: %v", err)
}
waitStarts(t, ch, 4)
}
// TestChannelSupervisorRestartReplacesRunningChannel pins the in-flight path: a
// restart of a channel whose run is live cancels the run and starts the
// replacement, with no backoff.
func TestChannelSupervisorRestartReplacesRunningChannel(t *testing.T) {
ch := &fakeChannel{} // blocks until its run context ends
srv := startService(t, ch, channelRetryPolicy{attempts: 3, backoff: time.Hour, maxBackoff: time.Hour})
waitStarts(t, ch, 1)
if err := srv.restartChannel("fake", ResolvedConfig{}); err != nil {
t.Fatalf("restart running channel: %v", err)
}
waitStarts(t, ch, 2)
}
// TestChannelRestartRefusedWithoutSupervisor pins that a channel with no
// supervisor -- never started, or already shut down -- is refused rather than
// signalling a goroutine that is gone.
func TestChannelRestartRefusedWithoutSupervisor(t *testing.T) {
ch := &fakeChannel{}
inst := &channelInstance{
name: "fake",
channel: ch,
rebuild: func(ResolvedConfig) (channels.Channel, error) { return ch, nil },
}
srv, err := compose(context.Background(), deps{Log: slog.Default(), ExtensionChannels: []*channelInstance{inst}})
if err != nil {
t.Fatalf("compose: %v", err)
}
if err := srv.restartChannel("fake", ResolvedConfig{}); err == nil {
t.Fatal("restart before the service started should be refused")
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
defer close(done)
_ = srv.Start(ctx)
}()
waitStarts(t, ch, 1)
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("service did not stop")
}
if err := srv.restartChannel("fake", ResolvedConfig{}); err == nil {
t.Fatal("restart after shutdown should be refused")
}
}