fix(archiemessaging): recover a channel that fails on its own
This commit is contained in:
3 files changed
+300
-18
No files matched your search
@@ -58,7 +58,7 @@ type deps struct {
|
||||
}
|
||||
|
||||
// channelInstance is one composed channel and what is needed to restart it.
|
||||
// mu guards channel and cancel.
|
||||
// mu guards every field below.
|
||||
type channelInstance struct {
|
||||
name string
|
||||
rebuild func(ResolvedConfig) (channels.Channel, error)
|
||||
@@ -68,6 +68,9 @@ type channelInstance struct {
|
||||
cancel context.CancelFunc
|
||||
supervised bool
|
||||
restartPending bool
|
||||
// notify wakes a supervisor that is between runs, so an explicit restart
|
||||
// starts the replacement instead of waiting out a backoff or a park.
|
||||
notify chan struct{}
|
||||
}
|
||||
|
||||
// current returns the channel instance currently serving, under the lock.
|
||||
@@ -108,6 +111,9 @@ type Service struct {
|
||||
appliedVersion atomic.Int64
|
||||
// reconcileInterval is how often the stored settings are re-read.
|
||||
reconcileInterval time.Duration
|
||||
// retry bounds how a supervisor retries a channel that fails on its own
|
||||
// before parking it for an explicit restart.
|
||||
retry channelRetryPolicy
|
||||
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
@@ -135,6 +141,7 @@ func compose(ctx context.Context, d deps) (*Service, error) {
|
||||
if srv.reconcileInterval <= 0 {
|
||||
srv.reconcileInterval = applystatus.RestampInterval
|
||||
}
|
||||
srv.retry = defaultChannelRetry()
|
||||
|
||||
instances, err := composeChannels(ctx, d)
|
||||
if err != nil {
|
||||
@@ -167,6 +174,9 @@ func composeChannels(ctx context.Context, d deps) ([]*channelInstance, error) {
|
||||
}
|
||||
instances = append(instances, instance)
|
||||
}
|
||||
for _, instance := range instances {
|
||||
instance.notify = make(chan struct{}, 1)
|
||||
}
|
||||
return instances, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -4,14 +4,21 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/channels"
|
||||
)
|
||||
|
||||
// runChannel runs one channel and starts its replacement after a requested
|
||||
// restart. A channel that exits on its own is not restarted.
|
||||
// runChannel supervises one channel for the service's lifetime. It starts the
|
||||
// channel and, when the run ends, decides what happens next: a requested
|
||||
// restart starts the replacement, a run that failed is retried with bounded
|
||||
// backoff, and a channel that keeps failing is parked until an explicit
|
||||
// restart. Only shutdown ends the supervisor, so a channel that stops is never
|
||||
// left without one to start it again.
|
||||
func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
|
||||
defer s.wg.Done()
|
||||
defer c.supervisorExits()
|
||||
attempts := 0
|
||||
for {
|
||||
c.mu.Lock()
|
||||
ch := c.channel
|
||||
@@ -24,7 +31,7 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
|
||||
|
||||
// Decide the next state under the same lock a restart takes, so a
|
||||
// restart racing this exit is either seen here (restartPending) or
|
||||
// observed as a channel that is no longer supervised.
|
||||
// delivered to the wait below (notify).
|
||||
c.mu.Lock()
|
||||
c.cancel = nil
|
||||
shutdown := ctx.Err() != nil
|
||||
@@ -32,9 +39,6 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
|
||||
if !shutdown && restart {
|
||||
c.restartPending = false
|
||||
}
|
||||
if shutdown || !restart {
|
||||
c.supervised = false
|
||||
}
|
||||
c.mu.Unlock()
|
||||
cancel()
|
||||
|
||||
@@ -52,23 +56,122 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
|
||||
return
|
||||
}
|
||||
if restart {
|
||||
// The replacement is already in c.channel (see restartChannel); the
|
||||
// next pass starts it.
|
||||
attempts = 0
|
||||
continue
|
||||
}
|
||||
// The channel stopped on its own.
|
||||
if err != nil {
|
||||
s.status.MarkFailed(c.name, err.Error())
|
||||
s.log.Error("channel stopped with error", "name", c.name, "err", err)
|
||||
} else {
|
||||
if err == nil {
|
||||
// The channel stopped itself. Hold it stopped rather than
|
||||
// spending the failure retry budget on a run that chose to end.
|
||||
s.status.MarkStopped(c.name, "")
|
||||
s.log.Info("channel stopped", "name", c.name)
|
||||
if s.awaitChannel(ctx, c, 0) == channelShutdown {
|
||||
return
|
||||
}
|
||||
attempts = 0
|
||||
continue
|
||||
}
|
||||
attempts++
|
||||
s.status.MarkFailed(c.name, err.Error())
|
||||
if attempts >= s.retry.attempts {
|
||||
s.log.Error("channel failed repeatedly; parked until an explicit restart",
|
||||
"name", c.name, "attempts", attempts, "err", err)
|
||||
if s.awaitChannel(ctx, c, 0) == channelShutdown {
|
||||
return
|
||||
}
|
||||
attempts = 0
|
||||
continue
|
||||
}
|
||||
backoff := s.retry.backoffFor(attempts)
|
||||
s.log.Warn("channel failed; restarting", "name", c.name,
|
||||
"attempt", attempts, "of", s.retry.attempts, "in", backoff, "err", err)
|
||||
switch s.awaitChannel(ctx, c, backoff) {
|
||||
case channelShutdown:
|
||||
return
|
||||
case channelRestart:
|
||||
attempts = 0
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// restartChannel builds a replacement for channel id from next, then stops
|
||||
// the old one and starts it. A channel not running is not restarted.
|
||||
// channelRetry bounds how a supervisor retries a channel that fails on its own
|
||||
// before parking it for an explicit restart.
|
||||
type channelRetryPolicy struct {
|
||||
attempts int
|
||||
backoff time.Duration
|
||||
maxBackoff time.Duration
|
||||
}
|
||||
|
||||
// defaultChannelRetry is the retry policy a composed service uses.
|
||||
func defaultChannelRetry() channelRetryPolicy {
|
||||
return channelRetryPolicy{attempts: 5, backoff: time.Second, maxBackoff: 30 * time.Second}
|
||||
}
|
||||
|
||||
// backoffFor is the wait before attempt n (1-based): doubling from backoff,
|
||||
// capped at maxBackoff so a long outage does not become a long silence between
|
||||
// retries.
|
||||
func (r channelRetryPolicy) backoffFor(attempt int) time.Duration {
|
||||
backoff := r.backoff << (attempt - 1)
|
||||
if backoff <= 0 || backoff > r.maxBackoff {
|
||||
return r.maxBackoff
|
||||
}
|
||||
return backoff
|
||||
}
|
||||
|
||||
// channelWait is how a supervisor's wait between runs ended.
|
||||
type channelWait int
|
||||
|
||||
const (
|
||||
// channelRetry: the backoff elapsed; run the channel again.
|
||||
channelRetry channelWait = iota
|
||||
// channelRestart: an explicit restart arrived; start the replacement.
|
||||
channelRestart
|
||||
// channelShutdown: the service is stopping.
|
||||
channelShutdown
|
||||
)
|
||||
|
||||
// awaitChannel blocks until an explicit restart arrives, the service stops, or
|
||||
// the backoff elapses. A delay of zero waits with no timer: the parked state
|
||||
// after a channel stopped or gave up, which only a restart or shutdown ends.
|
||||
func (s *Service) awaitChannel(ctx context.Context, c *channelInstance, delay time.Duration) channelWait {
|
||||
if delay <= 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return channelShutdown
|
||||
case <-c.notify:
|
||||
return channelRestart
|
||||
}
|
||||
}
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return channelShutdown
|
||||
case <-c.notify:
|
||||
return channelRestart
|
||||
case <-timer.C:
|
||||
// A restart delivered as the timer fired is read here rather than
|
||||
// lost between the two branches.
|
||||
select {
|
||||
case <-c.notify:
|
||||
return channelRestart
|
||||
default:
|
||||
return channelRetry
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// supervisorExits records that a channel's supervisor is leaving, so a restart
|
||||
// after shutdown is refused rather than signalling a goroutine that is gone.
|
||||
func (c *channelInstance) supervisorExits() {
|
||||
c.mu.Lock()
|
||||
c.supervised = false
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// restartChannel builds a replacement for channel id from next and starts it in
|
||||
// place of the running one. A supervisor that is between runs -- backing off a
|
||||
// failure, or parked after giving up -- is woken to start the replacement; a
|
||||
// channel with no supervisor (never started, or shut down) is refused.
|
||||
func (s *Service) restartChannel(id string, next ResolvedConfig) error {
|
||||
c := s.instanceByID(id)
|
||||
if c == nil {
|
||||
@@ -90,11 +193,24 @@ func (s *Service) restartChannel(id string, next ResolvedConfig) error {
|
||||
return fmt.Errorf("channel %q is not running", id)
|
||||
}
|
||||
c.channel = replacement
|
||||
c.restartPending = true
|
||||
cancel := c.cancel
|
||||
if cancel != nil {
|
||||
// A run is in flight. Cancelling its context ends Start, and the
|
||||
// supervisor sees restartPending and starts the replacement.
|
||||
c.restartPending = true
|
||||
} else {
|
||||
// The supervisor is between runs. Wake it so it starts the replacement
|
||||
// now, rather than waiting out its backoff or its park.
|
||||
c.restartPending = false
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case c.notify <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
package archiemessaging
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/channels"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
)
|
||||
|
||||
// fakeChannel is a channel whose runs a test scripts: it counts starts and
|
||||
// either fails immediately or blocks until its run context ends.
|
||||
type fakeChannel struct {
|
||||
mu sync.Mutex
|
||||
starts int
|
||||
runErr error
|
||||
}
|
||||
|
||||
func (f *fakeChannel) Name() string { return "fake" }
|
||||
|
||||
func (f *fakeChannel) Start(ctx context.Context, _ messaging.ChatContract, _ channels.Lifecycle) error {
|
||||
f.mu.Lock()
|
||||
f.starts++
|
||||
f.mu.Unlock()
|
||||
if f.runErr != nil {
|
||||
return f.runErr
|
||||
}
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeChannel) Stop(context.Context) error { return nil }
|
||||
|
||||
func (f *fakeChannel) ConfigSchema() json.RawMessage { return nil }
|
||||
|
||||
func (f *fakeChannel) ValidateConfig(map[string]any) error { return nil }
|
||||
|
||||
func (f *fakeChannel) count() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.starts
|
||||
}
|
||||
|
||||
// startService composes and starts a service over one scripted channel, with a
|
||||
// retry policy the test drives without waiting out real backoff.
|
||||
func startService(t *testing.T, ch *fakeChannel, retry channelRetryPolicy) *Service {
|
||||
t.Helper()
|
||||
inst := &channelInstance{
|
||||
name: "fake",
|
||||
channel: ch,
|
||||
rebuild: func(ResolvedConfig) (channels.Channel, error) { return ch, nil },
|
||||
}
|
||||
srv, err := compose(context.Background(), deps{Log: slog.Default(), ExtensionChannels: []*channelInstance{inst}})
|
||||
if err != nil {
|
||||
t.Fatalf("compose: %v", err)
|
||||
}
|
||||
srv.retry = retry
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
_ = srv.Start(ctx)
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("service did not stop after cancel")
|
||||
}
|
||||
})
|
||||
return srv
|
||||
}
|
||||
|
||||
func waitStarts(t *testing.T, ch *fakeChannel, want int) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if ch.count() >= want {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
t.Fatalf("channel started %d times, want at least %d", ch.count(), want)
|
||||
}
|
||||
|
||||
// TestChannelSupervisorRetriesAndRestarts pins that a channel that fails on its
|
||||
// own is retried with bounded backoff and, once the budget is spent, parked so
|
||||
// an explicit restart brings it back rather than being refused.
|
||||
func TestChannelSupervisorRetriesAndRestarts(t *testing.T) {
|
||||
ch := &fakeChannel{runErr: errors.New("transport down")}
|
||||
srv := startService(t, ch, channelRetryPolicy{attempts: 3, backoff: time.Millisecond, maxBackoff: time.Millisecond})
|
||||
|
||||
// The supervisor retries the failed run instead of abandoning the channel.
|
||||
waitStarts(t, ch, 3)
|
||||
|
||||
if err := srv.restartChannel("fake", ResolvedConfig{}); err != nil {
|
||||
t.Fatalf("restart after the retry budget was spent: %v", err)
|
||||
}
|
||||
waitStarts(t, ch, 4)
|
||||
}
|
||||
|
||||
// TestChannelSupervisorRestartReplacesRunningChannel pins the in-flight path: a
|
||||
// restart of a channel whose run is live cancels the run and starts the
|
||||
// replacement, with no backoff.
|
||||
func TestChannelSupervisorRestartReplacesRunningChannel(t *testing.T) {
|
||||
ch := &fakeChannel{} // blocks until its run context ends
|
||||
srv := startService(t, ch, channelRetryPolicy{attempts: 3, backoff: time.Hour, maxBackoff: time.Hour})
|
||||
|
||||
waitStarts(t, ch, 1)
|
||||
if err := srv.restartChannel("fake", ResolvedConfig{}); err != nil {
|
||||
t.Fatalf("restart running channel: %v", err)
|
||||
}
|
||||
waitStarts(t, ch, 2)
|
||||
}
|
||||
|
||||
// TestChannelRestartRefusedWithoutSupervisor pins that a channel with no
|
||||
// supervisor -- never started, or already shut down -- is refused rather than
|
||||
// signalling a goroutine that is gone.
|
||||
func TestChannelRestartRefusedWithoutSupervisor(t *testing.T) {
|
||||
ch := &fakeChannel{}
|
||||
inst := &channelInstance{
|
||||
name: "fake",
|
||||
channel: ch,
|
||||
rebuild: func(ResolvedConfig) (channels.Channel, error) { return ch, nil },
|
||||
}
|
||||
srv, err := compose(context.Background(), deps{Log: slog.Default(), ExtensionChannels: []*channelInstance{inst}})
|
||||
if err != nil {
|
||||
t.Fatalf("compose: %v", err)
|
||||
}
|
||||
if err := srv.restartChannel("fake", ResolvedConfig{}); err == nil {
|
||||
t.Fatal("restart before the service started should be refused")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
_ = srv.Start(ctx)
|
||||
}()
|
||||
waitStarts(t, ch, 1)
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("service did not stop")
|
||||
}
|
||||
if err := srv.restartChannel("fake", ResolvedConfig{}); err == nil {
|
||||
t.Fatal("restart after shutdown should be refused")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user