Merge pull request #874 from samcharles93/fix/per-identity-boot-sweep

fix(daemon): sweep the root forge and every identity at startup
This commit is contained in:
sam authored and GitHub committed 2026-09-19 02:25:27 +10:00
commit 9f4eae5f74
2 files changed
+182 -7

No files matched your search

+53 -7
View File
@@ -259,15 +259,61 @@ func (d *Daemon) Startup(ctx context.Context) error {
if n > 0 {
d.Log.Info("re-queued tasks left running by a previous daemon", "count", n)
}
if err := d.Forge.AcceptInvitations(ctx); err != nil {
d.Log.Warn("invitation sweep failed", "err", err)
}
for _, r := range d.Cfg.Get().Repos {
if err := d.Forge.VerifyPush(ctx, r.Owner, r.Name); err != nil {
d.Log.Warn("repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err)
d.sweepAccess(ctx)
return nil
}
// sweepTimeout bounds one forge's boot sweep. Startup gates the gateways and
// the run loop, and a forge client carries no request deadline of its own, so
// an unreachable host would otherwise hold boot open indefinitely.
const sweepTimeout = 2 * time.Minute
// sweepAccess accepts pending invitations and verifies push access for the
// root forge and for every configured identity's forge.
//
// The root is swept even in multi-identity mode. Run takes runIdentities
// there and never polls the root repo list, but polling is not the only way a
// task reaches a repo: dispatchBinding enqueues with an empty identity and
// resolveBindingRepo falls back to the root repo list, and forgeFor/repoFor
// resolve an empty identity to the root forge and root repos. Root targets
// stay reachable, so they stay worth verifying; a deployment with no root
// forge configured gets the noop client, whose sweep is silent.
//
// Each forge is swept in its own goroutine under its own deadline, mirroring
// Run's goroutine-per-identity isolation. Warnings alone isolate a failing
// sibling; they do not isolate a stalled one.
func (d *Daemon) sweepAccess(ctx context.Context) {
sweep := func(identity string, fg forge.Forge, repos []config.Repo) {
log := d.Log
if identity != "" {
log = log.With("identity", identity)
}
ctx, cancel := context.WithTimeout(ctx, sweepTimeout)
defer cancel()
if err := fg.AcceptInvitations(ctx); err != nil {
log.Warn("invitation sweep failed", "err", err)
}
for _, r := range repos {
if err := fg.VerifyPush(ctx, r.Owner, r.Name); err != nil {
log.Warn("repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err)
}
}
}
return nil
var wg sync.WaitGroup
wg.Add(1 + len(d.Identities))
go func() {
defer wg.Done()
sweep("", d.Forge, d.Cfg.Get().Repos)
}()
for _, id := range d.Identities {
go func() {
defer wg.Done()
sweep(id.Name, id.Forge, id.Repos)
}()
}
wg.Wait()
}
// Run polls until the context ends. When Identities is non-empty, each
+129
View File
@@ -8,6 +8,7 @@ import (
"os"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -1336,6 +1337,134 @@ func TestRepoForPrefersOwningIdentityRepoList(t *testing.T) {
}
}
// sweepForge records invitation sweeps and push verifications so a test can
// assert which forges Startup swept. It embeds testForge for the rest of the
// forge.Forge surface; those methods are unreachable from Startup. Startup
// sweeps every forge concurrently, so the recorders are mutex-guarded.
type sweepForge struct {
testForge
mu sync.Mutex
invitations int
verifiedRepos []string
}
func (f *sweepForge) AcceptInvitations(context.Context) error {
f.mu.Lock()
defer f.mu.Unlock()
f.invitations++
return nil
}
func (f *sweepForge) VerifyPush(_ context.Context, owner, repo string) error {
f.mu.Lock()
defer f.mu.Unlock()
f.verifiedRepos = append(f.verifiedRepos, owner+"/"+repo)
return nil
}
func (f *sweepForge) snapshot() (int, []string) {
f.mu.Lock()
defer f.mu.Unlock()
return f.invitations, slices.Clone(f.verifiedRepos)
}
// TestStartupSweepsEveryConfiguredForge pins the boot-time invitation sweep
// and push verification. Startup used to sweep only d.Forge and
// d.Cfg.Get().Repos, so in multi-identity mode per-identity invites were
// never accepted and per-identity repos never push-verified: a task from one
// of those repos failed at dispatch instead of warning at boot.
//
// The root forge is swept in both modes. Run never polls the root repo list
// in multi-identity mode, but dispatchBinding enqueues with an empty identity
// against resolveBindingRepo's root-repo fallback, and forgeFor/repoFor
// resolve an empty identity to the root forge and root repos, so root targets
// stay reachable there.
func TestStartupSweepsEveryConfiguredForge(t *testing.T) {
type want struct {
invitations int
repos []string
}
tests := []struct {
name string
rootRepos []config.Repo
identities []config.Repo // one identity per entry, keyed by owner
wantRoot want
wantIDs []want
}{
{
name: "single identity sweeps the root forge",
rootRepos: []config.Repo{{Owner: "root", Name: "repo"}},
wantRoot: want{invitations: 1, repos: []string{"root/repo"}},
},
{
name: "multi identity sweeps the root forge and every identity",
rootRepos: []config.Repo{{Owner: "root", Name: "repo"}},
identities: []config.Repo{
{Owner: "one", Name: "a"},
{Owner: "two", Name: "b"},
},
wantRoot: want{invitations: 1, repos: []string{"root/repo"}},
wantIDs: []want{
{invitations: 1, repos: []string{"one/a"}},
{invitations: 1, repos: []string{"two/b"}},
},
},
{
name: "identity with no repos still gets its invitations swept",
rootRepos: nil,
identities: []config.Repo{
{Owner: "one", Name: ""},
},
wantRoot: want{invitations: 1},
wantIDs: []want{{invitations: 1}},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
rootFg := &sweepForge{}
idForges := make([]*sweepForge, len(tc.identities))
runners := make([]*IdentityRunner, len(tc.identities))
for i, r := range tc.identities {
idForges[i] = &sweepForge{}
runner := &IdentityRunner{Name: r.Owner, Forge: idForges[i]}
if r.Name != "" {
runner.Repos = []config.Repo{r}
}
runners[i] = runner
}
d := &Daemon{
Cfg: config.NewHolder(config.Config{Repos: tc.rootRepos}),
Store: store.OpenTest(t),
Forge: rootFg,
Log: slog.New(slog.DiscardHandler),
Identities: runners,
}
if err := d.Startup(context.Background()); err != nil {
t.Fatalf("Startup: %v", err)
}
assertSwept(t, "root", rootFg, tc.wantRoot.invitations, tc.wantRoot.repos)
for i, w := range tc.wantIDs {
assertSwept(t, runners[i].Name, idForges[i], w.invitations, w.repos)
}
})
}
}
func assertSwept(t *testing.T, name string, fg *sweepForge, wantInvitations int, wantRepos []string) {
t.Helper()
invitations, repos := fg.snapshot()
if invitations != wantInvitations {
t.Errorf("%s AcceptInvitations = %d, want %d", name, invitations, wantInvitations)
}
if !slices.Equal(repos, wantRepos) {
t.Errorf("%s verified repos = %v, want %v", name, repos, wantRepos)
}
}
// mustCoreConn returns the raw NATS connection for tests that drive core-NATS
// subscriptions directly, failing the test if the client is not connected.
func mustCoreConn(t *testing.T, c *arnats.Client) *natsio.Conn {