fix(archiemessaging): recover a channel that fails on its own
This commit is contained in:
3 files changed
+300
-18
No files matched your search
@@ -58,7 +58,7 @@ type deps struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// channelInstance is one composed channel and what is needed to restart it.
|
// channelInstance is one composed channel and what is needed to restart it.
|
||||||
// mu guards channel and cancel.
|
// mu guards every field below.
|
||||||
type channelInstance struct {
|
type channelInstance struct {
|
||||||
name string
|
name string
|
||||||
rebuild func(ResolvedConfig) (channels.Channel, error)
|
rebuild func(ResolvedConfig) (channels.Channel, error)
|
||||||
@@ -68,6 +68,9 @@ type channelInstance struct {
|
|||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
supervised bool
|
supervised bool
|
||||||
restartPending bool
|
restartPending bool
|
||||||
|
// notify wakes a supervisor that is between runs, so an explicit restart
|
||||||
|
// starts the replacement instead of waiting out a backoff or a park.
|
||||||
|
notify chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// current returns the channel instance currently serving, under the lock.
|
// current returns the channel instance currently serving, under the lock.
|
||||||
@@ -108,6 +111,9 @@ type Service struct {
|
|||||||
appliedVersion atomic.Int64
|
appliedVersion atomic.Int64
|
||||||
// reconcileInterval is how often the stored settings are re-read.
|
// reconcileInterval is how often the stored settings are re-read.
|
||||||
reconcileInterval time.Duration
|
reconcileInterval time.Duration
|
||||||
|
// retry bounds how a supervisor retries a channel that fails on its own
|
||||||
|
// before parking it for an explicit restart.
|
||||||
|
retry channelRetryPolicy
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
running bool
|
running bool
|
||||||
@@ -135,6 +141,7 @@ func compose(ctx context.Context, d deps) (*Service, error) {
|
|||||||
if srv.reconcileInterval <= 0 {
|
if srv.reconcileInterval <= 0 {
|
||||||
srv.reconcileInterval = applystatus.RestampInterval
|
srv.reconcileInterval = applystatus.RestampInterval
|
||||||
}
|
}
|
||||||
|
srv.retry = defaultChannelRetry()
|
||||||
|
|
||||||
instances, err := composeChannels(ctx, d)
|
instances, err := composeChannels(ctx, d)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -167,6 +174,9 @@ func composeChannels(ctx context.Context, d deps) ([]*channelInstance, error) {
|
|||||||
}
|
}
|
||||||
instances = append(instances, instance)
|
instances = append(instances, instance)
|
||||||
}
|
}
|
||||||
|
for _, instance := range instances {
|
||||||
|
instance.notify = make(chan struct{}, 1)
|
||||||
|
}
|
||||||
return instances, nil
|
return instances, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,14 +4,21 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/samcharles93/archie-core/internal/channels"
|
"github.com/samcharles93/archie-core/internal/channels"
|
||||||
)
|
)
|
||||||
|
|
||||||
// runChannel runs one channel and starts its replacement after a requested
|
// runChannel supervises one channel for the service's lifetime. It starts the
|
||||||
// restart. A channel that exits on its own is not restarted.
|
// channel and, when the run ends, decides what happens next: a requested
|
||||||
|
// restart starts the replacement, a run that failed is retried with bounded
|
||||||
|
// backoff, and a channel that keeps failing is parked until an explicit
|
||||||
|
// restart. Only shutdown ends the supervisor, so a channel that stops is never
|
||||||
|
// left without one to start it again.
|
||||||
func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
|
func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
|
||||||
defer s.wg.Done()
|
defer s.wg.Done()
|
||||||
|
defer c.supervisorExits()
|
||||||
|
attempts := 0
|
||||||
for {
|
for {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
ch := c.channel
|
ch := c.channel
|
||||||
@@ -24,7 +31,7 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
|
|||||||
|
|
||||||
// Decide the next state under the same lock a restart takes, so a
|
// Decide the next state under the same lock a restart takes, so a
|
||||||
// restart racing this exit is either seen here (restartPending) or
|
// restart racing this exit is either seen here (restartPending) or
|
||||||
// observed as a channel that is no longer supervised.
|
// delivered to the wait below (notify).
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
c.cancel = nil
|
c.cancel = nil
|
||||||
shutdown := ctx.Err() != nil
|
shutdown := ctx.Err() != nil
|
||||||
@@ -32,9 +39,6 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
|
|||||||
if !shutdown && restart {
|
if !shutdown && restart {
|
||||||
c.restartPending = false
|
c.restartPending = false
|
||||||
}
|
}
|
||||||
if shutdown || !restart {
|
|
||||||
c.supervised = false
|
|
||||||
}
|
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
@@ -52,23 +56,122 @@ func (s *Service) runChannel(ctx context.Context, c *channelInstance) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if restart {
|
if restart {
|
||||||
// The replacement is already in c.channel (see restartChannel); the
|
attempts = 0
|
||||||
// next pass starts it.
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// The channel stopped on its own.
|
if err == nil {
|
||||||
if err != nil {
|
// The channel stopped itself. Hold it stopped rather than
|
||||||
s.status.MarkFailed(c.name, err.Error())
|
// spending the failure retry budget on a run that chose to end.
|
||||||
s.log.Error("channel stopped with error", "name", c.name, "err", err)
|
|
||||||
} else {
|
|
||||||
s.status.MarkStopped(c.name, "")
|
s.status.MarkStopped(c.name, "")
|
||||||
|
s.log.Info("channel stopped", "name", c.name)
|
||||||
|
if s.awaitChannel(ctx, c, 0) == channelShutdown {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
attempts = 0
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
attempts++
|
||||||
|
s.status.MarkFailed(c.name, err.Error())
|
||||||
|
if attempts >= s.retry.attempts {
|
||||||
|
s.log.Error("channel failed repeatedly; parked until an explicit restart",
|
||||||
|
"name", c.name, "attempts", attempts, "err", err)
|
||||||
|
if s.awaitChannel(ctx, c, 0) == channelShutdown {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
attempts = 0
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
backoff := s.retry.backoffFor(attempts)
|
||||||
|
s.log.Warn("channel failed; restarting", "name", c.name,
|
||||||
|
"attempt", attempts, "of", s.retry.attempts, "in", backoff, "err", err)
|
||||||
|
switch s.awaitChannel(ctx, c, backoff) {
|
||||||
|
case channelShutdown:
|
||||||
|
return
|
||||||
|
case channelRestart:
|
||||||
|
attempts = 0
|
||||||
}
|
}
|
||||||
return
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// restartChannel builds a replacement for channel id from next, then stops
|
// channelRetry bounds how a supervisor retries a channel that fails on its own
|
||||||
// the old one and starts it. A channel not running is not restarted.
|
// before parking it for an explicit restart.
|
||||||
|
type channelRetryPolicy struct {
|
||||||
|
attempts int
|
||||||
|
backoff time.Duration
|
||||||
|
maxBackoff time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultChannelRetry is the retry policy a composed service uses.
|
||||||
|
func defaultChannelRetry() channelRetryPolicy {
|
||||||
|
return channelRetryPolicy{attempts: 5, backoff: time.Second, maxBackoff: 30 * time.Second}
|
||||||
|
}
|
||||||
|
|
||||||
|
// backoffFor is the wait before attempt n (1-based): doubling from backoff,
|
||||||
|
// capped at maxBackoff so a long outage does not become a long silence between
|
||||||
|
// retries.
|
||||||
|
func (r channelRetryPolicy) backoffFor(attempt int) time.Duration {
|
||||||
|
backoff := r.backoff << (attempt - 1)
|
||||||
|
if backoff <= 0 || backoff > r.maxBackoff {
|
||||||
|
return r.maxBackoff
|
||||||
|
}
|
||||||
|
return backoff
|
||||||
|
}
|
||||||
|
|
||||||
|
// channelWait is how a supervisor's wait between runs ended.
|
||||||
|
type channelWait int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// channelRetry: the backoff elapsed; run the channel again.
|
||||||
|
channelRetry channelWait = iota
|
||||||
|
// channelRestart: an explicit restart arrived; start the replacement.
|
||||||
|
channelRestart
|
||||||
|
// channelShutdown: the service is stopping.
|
||||||
|
channelShutdown
|
||||||
|
)
|
||||||
|
|
||||||
|
// awaitChannel blocks until an explicit restart arrives, the service stops, or
|
||||||
|
// the backoff elapses. A delay of zero waits with no timer: the parked state
|
||||||
|
// after a channel stopped or gave up, which only a restart or shutdown ends.
|
||||||
|
func (s *Service) awaitChannel(ctx context.Context, c *channelInstance, delay time.Duration) channelWait {
|
||||||
|
if delay <= 0 {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return channelShutdown
|
||||||
|
case <-c.notify:
|
||||||
|
return channelRestart
|
||||||
|
}
|
||||||
|
}
|
||||||
|
timer := time.NewTimer(delay)
|
||||||
|
defer timer.Stop()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return channelShutdown
|
||||||
|
case <-c.notify:
|
||||||
|
return channelRestart
|
||||||
|
case <-timer.C:
|
||||||
|
// A restart delivered as the timer fired is read here rather than
|
||||||
|
// lost between the two branches.
|
||||||
|
select {
|
||||||
|
case <-c.notify:
|
||||||
|
return channelRestart
|
||||||
|
default:
|
||||||
|
return channelRetry
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// supervisorExits records that a channel's supervisor is leaving, so a restart
|
||||||
|
// after shutdown is refused rather than signalling a goroutine that is gone.
|
||||||
|
func (c *channelInstance) supervisorExits() {
|
||||||
|
c.mu.Lock()
|
||||||
|
c.supervised = false
|
||||||
|
c.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// restartChannel builds a replacement for channel id from next and starts it in
|
||||||
|
// place of the running one. A supervisor that is between runs -- backing off a
|
||||||
|
// failure, or parked after giving up -- is woken to start the replacement; a
|
||||||
|
// channel with no supervisor (never started, or shut down) is refused.
|
||||||
func (s *Service) restartChannel(id string, next ResolvedConfig) error {
|
func (s *Service) restartChannel(id string, next ResolvedConfig) error {
|
||||||
c := s.instanceByID(id)
|
c := s.instanceByID(id)
|
||||||
if c == nil {
|
if c == nil {
|
||||||
@@ -90,11 +193,24 @@ func (s *Service) restartChannel(id string, next ResolvedConfig) error {
|
|||||||
return fmt.Errorf("channel %q is not running", id)
|
return fmt.Errorf("channel %q is not running", id)
|
||||||
}
|
}
|
||||||
c.channel = replacement
|
c.channel = replacement
|
||||||
c.restartPending = true
|
|
||||||
cancel := c.cancel
|
cancel := c.cancel
|
||||||
|
if cancel != nil {
|
||||||
|
// A run is in flight. Cancelling its context ends Start, and the
|
||||||
|
// supervisor sees restartPending and starts the replacement.
|
||||||
|
c.restartPending = true
|
||||||
|
} else {
|
||||||
|
// The supervisor is between runs. Wake it so it starts the replacement
|
||||||
|
// now, rather than waiting out its backoff or its park.
|
||||||
|
c.restartPending = false
|
||||||
|
}
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
if cancel != nil {
|
if cancel != nil {
|
||||||
cancel()
|
cancel()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case c.notify <- struct{}{}:
|
||||||
|
default:
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,156 @@
|
|||||||
|
package archiemessaging
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/channels"
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeChannel is a channel whose runs a test scripts: it counts starts and
|
||||||
|
// either fails immediately or blocks until its run context ends.
|
||||||
|
type fakeChannel struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
starts int
|
||||||
|
runErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeChannel) Name() string { return "fake" }
|
||||||
|
|
||||||
|
func (f *fakeChannel) Start(ctx context.Context, _ messaging.ChatContract, _ channels.Lifecycle) error {
|
||||||
|
f.mu.Lock()
|
||||||
|
f.starts++
|
||||||
|
f.mu.Unlock()
|
||||||
|
if f.runErr != nil {
|
||||||
|
return f.runErr
|
||||||
|
}
|
||||||
|
<-ctx.Done()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeChannel) Stop(context.Context) error { return nil }
|
||||||
|
|
||||||
|
func (f *fakeChannel) ConfigSchema() json.RawMessage { return nil }
|
||||||
|
|
||||||
|
func (f *fakeChannel) ValidateConfig(map[string]any) error { return nil }
|
||||||
|
|
||||||
|
func (f *fakeChannel) count() int {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return f.starts
|
||||||
|
}
|
||||||
|
|
||||||
|
// startService composes and starts a service over one scripted channel, with a
|
||||||
|
// retry policy the test drives without waiting out real backoff.
|
||||||
|
func startService(t *testing.T, ch *fakeChannel, retry channelRetryPolicy) *Service {
|
||||||
|
t.Helper()
|
||||||
|
inst := &channelInstance{
|
||||||
|
name: "fake",
|
||||||
|
channel: ch,
|
||||||
|
rebuild: func(ResolvedConfig) (channels.Channel, error) { return ch, nil },
|
||||||
|
}
|
||||||
|
srv, err := compose(context.Background(), deps{Log: slog.Default(), ExtensionChannels: []*channelInstance{inst}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compose: %v", err)
|
||||||
|
}
|
||||||
|
srv.retry = retry
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
_ = srv.Start(ctx)
|
||||||
|
}()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Error("service did not stop after cancel")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitStarts(t *testing.T, ch *fakeChannel, want int) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if ch.count() >= want {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("channel started %d times, want at least %d", ch.count(), want)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestChannelSupervisorRetriesAndRestarts pins that a channel that fails on its
|
||||||
|
// own is retried with bounded backoff and, once the budget is spent, parked so
|
||||||
|
// an explicit restart brings it back rather than being refused.
|
||||||
|
func TestChannelSupervisorRetriesAndRestarts(t *testing.T) {
|
||||||
|
ch := &fakeChannel{runErr: errors.New("transport down")}
|
||||||
|
srv := startService(t, ch, channelRetryPolicy{attempts: 3, backoff: time.Millisecond, maxBackoff: time.Millisecond})
|
||||||
|
|
||||||
|
// The supervisor retries the failed run instead of abandoning the channel.
|
||||||
|
waitStarts(t, ch, 3)
|
||||||
|
|
||||||
|
if err := srv.restartChannel("fake", ResolvedConfig{}); err != nil {
|
||||||
|
t.Fatalf("restart after the retry budget was spent: %v", err)
|
||||||
|
}
|
||||||
|
waitStarts(t, ch, 4)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestChannelSupervisorRestartReplacesRunningChannel pins the in-flight path: a
|
||||||
|
// restart of a channel whose run is live cancels the run and starts the
|
||||||
|
// replacement, with no backoff.
|
||||||
|
func TestChannelSupervisorRestartReplacesRunningChannel(t *testing.T) {
|
||||||
|
ch := &fakeChannel{} // blocks until its run context ends
|
||||||
|
srv := startService(t, ch, channelRetryPolicy{attempts: 3, backoff: time.Hour, maxBackoff: time.Hour})
|
||||||
|
|
||||||
|
waitStarts(t, ch, 1)
|
||||||
|
if err := srv.restartChannel("fake", ResolvedConfig{}); err != nil {
|
||||||
|
t.Fatalf("restart running channel: %v", err)
|
||||||
|
}
|
||||||
|
waitStarts(t, ch, 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestChannelRestartRefusedWithoutSupervisor pins that a channel with no
|
||||||
|
// supervisor -- never started, or already shut down -- is refused rather than
|
||||||
|
// signalling a goroutine that is gone.
|
||||||
|
func TestChannelRestartRefusedWithoutSupervisor(t *testing.T) {
|
||||||
|
ch := &fakeChannel{}
|
||||||
|
inst := &channelInstance{
|
||||||
|
name: "fake",
|
||||||
|
channel: ch,
|
||||||
|
rebuild: func(ResolvedConfig) (channels.Channel, error) { return ch, nil },
|
||||||
|
}
|
||||||
|
srv, err := compose(context.Background(), deps{Log: slog.Default(), ExtensionChannels: []*channelInstance{inst}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compose: %v", err)
|
||||||
|
}
|
||||||
|
if err := srv.restartChannel("fake", ResolvedConfig{}); err == nil {
|
||||||
|
t.Fatal("restart before the service started should be refused")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
_ = srv.Start(ctx)
|
||||||
|
}()
|
||||||
|
waitStarts(t, ch, 1)
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("service did not stop")
|
||||||
|
}
|
||||||
|
if err := srv.restartChannel("fake", ResolvedConfig{}); err == nil {
|
||||||
|
t.Fatal("restart after shutdown should be refused")
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user