fix(messaging): refuse failed channels as enablement success
This commit is contained in:
2 files changed
+68
-2
No files matched your search
@@ -321,12 +321,28 @@ const channelApplyPoll = 10 * time.Millisecond
|
||||
|
||||
// awaitChannelActive waits, up to channelApplyTimeout, for a channel to leave
|
||||
// or reach the stopped state, so an enable or disable is reported applied only
|
||||
// once the channel is actually running (or not) rather than merely queued.
|
||||
// once the channel is actually running (or not) rather than merely queued. A
|
||||
// failed channel is never active: reporting it so would mark the settings
|
||||
// version applied and retire a failure the next tick must retry. Only a
|
||||
// serving state counts, and a failed enable fails fast instead of burning the
|
||||
// timeout while no restart can succeed without one.
|
||||
func (s *Service) awaitChannelActive(id string, active bool) error {
|
||||
deadline := time.Now().Add(channelApplyTimeout)
|
||||
for {
|
||||
for _, st := range s.status.Snapshot() {
|
||||
if st.ID == id && (st.State != status.StateStopped) == active {
|
||||
if st.ID != id {
|
||||
continue
|
||||
}
|
||||
if active && st.State == status.StateFailed {
|
||||
return fmt.Errorf("channel %q failed to start", id)
|
||||
}
|
||||
// Serving counts, impaired or not; anything else is still in
|
||||
// flight, except a stopped channel answering a disable.
|
||||
if st.State == status.StateRunning || st.State == status.StateDegraded {
|
||||
if active {
|
||||
return nil
|
||||
}
|
||||
} else if !active && st.State == status.StateStopped {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/channels"
|
||||
"github.com/samcharles93/archie-core/internal/channels/status"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
)
|
||||
|
||||
@@ -92,6 +93,55 @@ func waitStarts(t *testing.T, ch *fakeChannel, want int) {
|
||||
t.Fatalf("channel started %d times, want at least %d", ch.count(), want)
|
||||
}
|
||||
|
||||
// TestAwaitChannelActiveStates pins what counts as up: running and degraded
|
||||
// satisfy an enable, only stopped satisfies a disable, and a failed enable
|
||||
// fails fast instead of reporting success the next tick would never retry.
|
||||
func TestAwaitChannelActiveStates(t *testing.T) {
|
||||
mark := map[string]func(*status.Manager){
|
||||
"running": func(m *status.Manager) { m.MarkRunning("fake") },
|
||||
"degraded": func(m *status.Manager) { m.MarkDegraded("fake", "slow") },
|
||||
"failed": func(m *status.Manager) { m.MarkFailed("fake", "boom") },
|
||||
"starting": func(m *status.Manager) { m.MarkStarting("fake") },
|
||||
}
|
||||
// Refusals that wait out the 5s timeout (disable while failed or
|
||||
// running) are pre-existing behavior, unchanged here, and too slow to
|
||||
// pin; every case below answers at once.
|
||||
tests := []struct {
|
||||
name string
|
||||
state string
|
||||
active bool
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "enable accepts running", state: "running", active: true},
|
||||
{name: "enable accepts degraded", state: "degraded", active: true},
|
||||
{name: "enable refuses failed", state: "failed", active: true, wantErr: true},
|
||||
{name: "disable accepts stopped", state: "", active: false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
manager := status.NewManager([]status.Descriptor{{ID: "fake"}})
|
||||
if tt.state == "" {
|
||||
manager.MarkStopped("fake", "")
|
||||
} else {
|
||||
mark[tt.state](manager)
|
||||
}
|
||||
srv := &Service{status: manager}
|
||||
// Every case above answers at once: success and failed-enable
|
||||
// refusal never reach the timeout wait.
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- srv.awaitChannelActive("fake", tt.active) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("err = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
case <-time.After(6 * time.Second):
|
||||
t.Fatal("await did not answer")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestChannelSupervisorRetriesAndRestarts pins that a channel that fails on its
|
||||
// own is retried with bounded backoff and, once the budget is spent, parked so
|
||||
// an explicit restart brings it back rather than being refused.
|
||||
|
||||
Reference in new issue
Block a user