diff --git a/internal/app/archiemessaging/supervise.go b/internal/app/archiemessaging/supervise.go index e1813fad4..d370c451a 100644 --- a/internal/app/archiemessaging/supervise.go +++ b/internal/app/archiemessaging/supervise.go @@ -321,12 +321,28 @@ const channelApplyPoll = 10 * time.Millisecond // awaitChannelActive waits, up to channelApplyTimeout, for a channel to leave // or reach the stopped state, so an enable or disable is reported applied only -// once the channel is actually running (or not) rather than merely queued. +// once the channel is actually running (or not) rather than merely queued. A +// failed channel is never active: reporting it so would mark the settings +// version applied and retire a failure the next tick must retry. Only a +// serving state counts, and a failed enable fails fast instead of burning the +// timeout while no restart can succeed without one. func (s *Service) awaitChannelActive(id string, active bool) error { deadline := time.Now().Add(channelApplyTimeout) for { for _, st := range s.status.Snapshot() { - if st.ID == id && (st.State != status.StateStopped) == active { + if st.ID != id { + continue + } + if active && st.State == status.StateFailed { + return fmt.Errorf("channel %q failed to start", id) + } + // Serving counts, impaired or not; anything else is still in + // flight, except a stopped channel answering a disable. + if st.State == status.StateRunning || st.State == status.StateDegraded { + if active { + return nil + } + } else if !active && st.State == status.StateStopped { return nil } } diff --git a/internal/app/archiemessaging/supervise_test.go b/internal/app/archiemessaging/supervise_test.go index c748a959e..72aec1145 100644 --- a/internal/app/archiemessaging/supervise_test.go +++ b/internal/app/archiemessaging/supervise_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/samcharles93/archie-core/internal/channels" + "github.com/samcharles93/archie-core/internal/channels/status" "github.com/samcharles93/archie-core/internal/domain/messaging" ) @@ -92,6 +93,55 @@ func waitStarts(t *testing.T, ch *fakeChannel, want int) { t.Fatalf("channel started %d times, want at least %d", ch.count(), want) } +// TestAwaitChannelActiveStates pins what counts as up: running and degraded +// satisfy an enable, only stopped satisfies a disable, and a failed enable +// fails fast instead of reporting success the next tick would never retry. +func TestAwaitChannelActiveStates(t *testing.T) { + mark := map[string]func(*status.Manager){ + "running": func(m *status.Manager) { m.MarkRunning("fake") }, + "degraded": func(m *status.Manager) { m.MarkDegraded("fake", "slow") }, + "failed": func(m *status.Manager) { m.MarkFailed("fake", "boom") }, + "starting": func(m *status.Manager) { m.MarkStarting("fake") }, + } + // Refusals that wait out the 5s timeout (disable while failed or + // running) are pre-existing behavior, unchanged here, and too slow to + // pin; every case below answers at once. + tests := []struct { + name string + state string + active bool + wantErr bool + }{ + {name: "enable accepts running", state: "running", active: true}, + {name: "enable accepts degraded", state: "degraded", active: true}, + {name: "enable refuses failed", state: "failed", active: true, wantErr: true}, + {name: "disable accepts stopped", state: "", active: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manager := status.NewManager([]status.Descriptor{{ID: "fake"}}) + if tt.state == "" { + manager.MarkStopped("fake", "") + } else { + mark[tt.state](manager) + } + srv := &Service{status: manager} + // Every case above answers at once: success and failed-enable + // refusal never reach the timeout wait. + done := make(chan error, 1) + go func() { done <- srv.awaitChannelActive("fake", tt.active) }() + select { + case err := <-done: + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) + } + case <-time.After(6 * time.Second): + t.Fatal("await did not answer") + } + }) + } +} + // TestChannelSupervisorRetriesAndRestarts pins that a channel that fails on its // own is retried with bounded backoff and, once the budget is spent, parked so // an explicit restart brings it back rather than being refused.