From eeeb88d565f89bbc0f9ffbcff1004043bc1aaaa3 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 17 Oct 2026 10:18:46 +1100 Subject: [PATCH] feat(archied): dispatch every organisation's schedules --- internal/app/archied/scheduling.go | 154 +++++++++++++++--- internal/infrastructure/staterpc/orgs.go | 24 ++- internal/infrastructure/staterpc/orgs_test.go | 8 +- 3 files changed, 159 insertions(+), 27 deletions(-) diff --git a/internal/app/archied/scheduling.go b/internal/app/archied/scheduling.go index fa3e934f9..0e1dbb3d0 100644 --- a/internal/app/archied/scheduling.go +++ b/internal/app/archied/scheduling.go @@ -5,9 +5,12 @@ import ( "errors" "fmt" "log/slog" + "strings" "time" "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/domain/access" + "github.com/samcharles93/archie-core/internal/domain/org" "github.com/samcharles93/archie-core/internal/domain/scheduling" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/gateway" @@ -60,7 +63,10 @@ func (b *boot) setupScheduling() error { return fmt.Errorf("scheduling: %w", err) } - store := scheduleResourceStore{client: b.controlPlane} + store := scheduleResourceStore{client: b.controlPlane, log: b.log} + if orgs, ok := b.stateStore.(orgDirectory); ok { + store.orgs = orgs + } chatRunner, err := crondelivery.NewChatCourier(store, func(ctx context.Context, target scheduling.Target, text string) error { return b.messaging.Deliver(ctx, target.Channel, target.ChatID, text) @@ -68,13 +74,22 @@ func (b *boot) setupScheduling() error { if err != nil { return fmt.Errorf("scheduling: build chat runner: %w", err) } - runners := map[string]scheduling.Runner{scheduling.KindChat: chatRunner} + // Channels are instance-owned, so only the system org may message them. + runners := map[string]scheduling.Runner{scheduling.KindChat: systemOrgOnly{ + next: chatRunner, store: store, + refusal: errors.New("chat schedules need an organisation channel; use a workflow schedule"), + }} if b.chatTasks != nil { workflowRunner, err := crondelivery.NewWorkflowTask(store, spawnTaskSubmitter{creator: b.chatTasks, identity: b.defaultChatIdentity}) if err != nil { return fmt.Errorf("scheduling: build workflow runner: %w", err) } - runners[scheduling.KindWorkflow] = workflowRunner + // The chat identity profiles that submit scheduled work belong to the + // system org; another org's schedule must name the agent it runs as. + runners[scheduling.KindWorkflow] = systemOrgOnly{ + next: workflowRunner, store: store, + refusal: errors.New("workflow schedules run only in the system organisation until a schedule names its agent"), + } } router, err := crondelivery.NewRouter(store, runners) if err != nil { @@ -90,22 +105,73 @@ func (b *boot) setupScheduling() error { return nil } +// orgDirectory finds every organisation and an agent principal to act as in +// each, because the control plane serves a caller only its own org. +type orgDirectory interface { + access.PrincipalSource + ListOrgs(ctx context.Context) ([]org.Org, error) + ListOrgAgents(ctx context.Context, id org.OrgID) ([]org.AgentAssignment, error) +} + +// scheduleResourceStore is every organisation's schedules document. Engine job +// ids are "/" so one engine ticks all orgs, and each lookup or +// run record acts under the owning org's principal. type scheduleResourceStore struct { client *controlplane.Client + orgs orgDirectory + log *slog.Logger } -// The schedules document behind the control plane is the production -// RouterStore: it resolves a job id to its spec and records a completed run. -// The assertions this replaces used to pin the retired file-backed store; -// crondelivery's contract comments name this type as the implementation. var ( _ crondelivery.SpecLookup = scheduleResourceStore{} _ crondelivery.RunRecorder = scheduleResourceStore{} _ crondelivery.RouterStore = scheduleResourceStore{} ) -func (s scheduleResourceStore) Get(ctx context.Context, id string) (scheduling.JobSpec, bool, error) { - jobs, _, err := s.client.Schedules(ctx) +func jobKey(owner org.OrgID, id string) string { return string(owner) + "/" + id } + +func splitJobKey(key string) (org.OrgID, string) { + owner, id, ok := strings.Cut(key, "/") + if !ok { + return org.DefaultOrgID, key + } + return org.OrgID(owner), id +} + +// orgContext returns ctx acting as owner. The system org is the daemon's own, +// so it needs no agent principal. +func (s scheduleResourceStore) orgContext(ctx context.Context, owner org.OrgID) (context.Context, error) { + if owner == org.DefaultOrgID { + return ctx, nil + } + if s.orgs == nil { + return nil, fmt.Errorf("organisation %q: no organisation directory", owner) + } + agents, err := s.orgs.ListOrgAgents(ctx, owner) + if err != nil { + return nil, fmt.Errorf("organisation %q: list agents: %w", owner, err) + } + for _, agent := range agents { + principal, err := s.orgs.PrincipalFor(ctx, agent.IdentityID) + if err == nil && principal.Org == owner { + return org.WithOrg(access.WithPrincipal(ctx, principal), owner), nil + } + } + return nil, fmt.Errorf("organisation %q has no agent to run its schedules as", owner) +} + +func (s scheduleResourceStore) orgSchedules(ctx context.Context, key string) (context.Context, string, []scheduling.JobSpec, int64, error) { + owner, id := splitJobKey(key) + octx, err := s.orgContext(ctx, owner) + if err != nil { + return nil, "", nil, 0, err + } + jobs, version, err := s.client.Schedules(octx) + return octx, id, jobs, version, err +} + +func (s scheduleResourceStore) Get(ctx context.Context, key string) (scheduling.JobSpec, bool, error) { + _, id, jobs, _, err := s.orgSchedules(ctx, key) if err != nil { return scheduling.JobSpec{}, false, err } @@ -117,23 +183,69 @@ func (s scheduleResourceStore) Get(ctx context.Context, id string) (scheduling.J return scheduling.JobSpec{}, false, nil } -func (s scheduleResourceStore) Due(ctx context.Context, now time.Time) ([]scheduling.Job, error) { - jobs, _, err := s.client.Schedules(ctx) - if err != nil { - return nil, err +// owners lists every org, system org first. Without the directory the system +// org still runs. +func (s scheduleResourceStore) owners(ctx context.Context) []org.OrgID { + owners := []org.OrgID{org.DefaultOrgID} + if s.orgs == nil { + return owners } - due := make([]scheduling.Job, 0, len(jobs)) - for _, job := range jobs { - if !job.NextRun.After(now) { - due = append(due, scheduling.Job{ID: job.ID, Pool: scheduling.Pool(job.Pool), Detail: job.Detail}) + list, err := s.orgs.ListOrgs(ctx) + if err != nil { + s.log.Warn("scheduling: organisations unavailable; running system schedules only", "err", err) + return owners + } + for _, o := range list { + if o.ID != org.DefaultOrgID { + owners = append(owners, o.ID) + } + } + return owners +} + +func (s scheduleResourceStore) Due(ctx context.Context, now time.Time) ([]scheduling.Job, error) { + var due []scheduling.Job + for _, owner := range s.owners(ctx) { + octx, err := s.orgContext(ctx, owner) + if err != nil { + s.log.Warn("scheduling: skipping organisation", "org", owner, "err", err) + continue + } + jobs, _, err := s.client.Schedules(octx) + if err != nil { + if owner == org.DefaultOrgID { + return nil, err + } + s.log.Warn("scheduling: organisation schedules unavailable", "org", owner, "err", err) + continue + } + for _, job := range jobs { + if !job.NextRun.After(now) { + due = append(due, scheduling.Job{ID: jobKey(owner, job.ID), Pool: scheduling.Pool(job.Pool), Detail: job.Detail}) + } } } return due, nil } -func (s scheduleResourceStore) MarkRun(ctx context.Context, id string, runAt time.Time) error { +// systemOrgOnly runs a kind for the system org and refuses it for any other, +// recording the run so the refused job is not retried every tick. +type systemOrgOnly struct { + next scheduling.Runner + store scheduleResourceStore + refusal error +} + +func (r systemOrgOnly) Run(ctx context.Context, job scheduling.Job) error { + if owner, _ := splitJobKey(job.ID); owner == org.DefaultOrgID { + return r.next.Run(ctx, job) + } + return errors.Join(fmt.Errorf("schedule %q: %w", job.ID, r.refusal), r.store.MarkRun(ctx, job.ID, time.Now())) +} + +func (s scheduleResourceStore) MarkRun(ctx context.Context, key string, runAt time.Time) error { for range 4 { - jobs, version, err := s.client.Schedules(ctx) + octx, id, jobs, version, err := s.orgSchedules(ctx, key) if err != nil { return err } @@ -157,9 +269,9 @@ func (s scheduleResourceStore) MarkRun(ctx context.Context, id string, runAt tim break } if !found { - return fmt.Errorf("%w: id %q", scheduling.ErrJobNotFound, id) + return fmt.Errorf("%w: id %q", scheduling.ErrJobNotFound, key) } - _, err = s.client.ReplaceSchedules(ctx, jobs, version, "system:scheduler", "scheduler", fmt.Sprintf("schedule-run:%s:%d", id, runAt.UnixNano())) + _, err = s.client.ReplaceSchedules(octx, jobs, version, "system:scheduler", "scheduler", fmt.Sprintf("schedule-run:%s:%d", key, runAt.UnixNano())) if err == nil { return nil } diff --git a/internal/infrastructure/staterpc/orgs.go b/internal/infrastructure/staterpc/orgs.go index 3e6f03e14..b54a8108b 100644 --- a/internal/infrastructure/staterpc/orgs.go +++ b/internal/infrastructure/staterpc/orgs.go @@ -71,8 +71,9 @@ func (s *server) ListOrgs(ctx context.Context, _ *pb.ListOrgsRequest) (*pb.ListO if err != nil { return nil, s.orgErr("ListOrgs", err) } - // Only an instance admin sees every org; anyone else sees their own. - if p, ok := access.PrincipalFromContext(ctx); !ok || !p.InstanceAdmin() { + // Only an instance admin or an internal service sees every org; anyone + // else sees their own. + if !readsEveryOrg(ctx) { caller := org.OrgFromContext(ctx) values = slices.DeleteFunc(values, func(o org.Org) bool { return o.ID != caller }) } @@ -83,8 +84,17 @@ func (s *server) ListOrgs(ctx context.Context, _ *pb.ListOrgsRequest) (*pb.ListO return &pb.ListOrgsResponse{Orgs: out}, nil } -// orgInForce is the org a request is served for: the one the caller's -// principal carries. A request field cannot choose another tenant. +// readsEveryOrg reports whether ctx may read another org's directory: an +// instance admin, or an internal service acting for no one and confined to no +// org, such as the daemon ticking every org's schedules. +func readsEveryOrg(ctx context.Context) bool { + if p, ok := access.PrincipalFromContext(ctx); ok { + return p.InstanceAdmin() + } + _, scoped := org.Scope(ctx) + return !scoped +} + // orgFor is the org a request acts in: the caller's own, or the one it names // when the caller is an instance admin. Anyone else naming another org still // acts in their own, so the contract cannot reach another tenant. @@ -221,7 +231,11 @@ func (s *server) ListOrgAgents(ctx context.Context, r *pb.ListOrgAgentsRequest) if s.deps.Orgs == nil { return nil, errOrgsUnavailable } - values, err := s.deps.Orgs.ListOrgAgents(ctx, orgFor(ctx, r.GetOrgId())) + owner := orgFor(ctx, r.GetOrgId()) + if r.GetOrgId() != "" && readsEveryOrg(ctx) { + owner = org.OrgID(r.GetOrgId()) + } + values, err := s.deps.Orgs.ListOrgAgents(ctx, owner) if err != nil { return nil, s.orgErr("ListOrgAgents", err) } diff --git a/internal/infrastructure/staterpc/orgs_test.go b/internal/infrastructure/staterpc/orgs_test.go index 64f93c175..f178a2b47 100644 --- a/internal/infrastructure/staterpc/orgs_test.go +++ b/internal/infrastructure/staterpc/orgs_test.go @@ -78,7 +78,7 @@ func (*fakeOrgs) OrgForIdentity(context.Context, identity.IdentityID) (org.OrgID func TestOrgRPCsActInTheCallersOrg(t *testing.T) { repo := &fakeOrgs{} s := &server{deps: Deps{Orgs: repo, Log: slog.New(slog.DiscardHandler)}} - ctx := org.WithOrg(context.Background(), "acme") + ctx := org.WithScope(org.WithOrg(context.Background(), "acme"), "acme") got, err := s.GetOrg(ctx, &pb.GetOrgRequest{OrgId: "other"}) if err != nil || got.GetOrg().GetId() != "acme" { @@ -97,6 +97,12 @@ func TestOrgRPCsActInTheCallersOrg(t *testing.T) { if err != nil || len(agents.GetAgents()) != 1 || agents.Agents[0].OrgId != "acme" { t.Fatalf("assignment read escaped org: %v %v", agents, err) } + // An unscoped internal service, the daemon ticking every org's + // schedules, reads the org it names. + agents, err = s.ListOrgAgents(context.Background(), &pb.ListOrgAgentsRequest{OrgId: "acme"}) + if err != nil || len(agents.GetAgents()) != 1 { + t.Fatalf("internal service could not read a named org: %v %v", agents, err) + } if _, err := s.CreateWorkspace(ctx, &pb.CreateWorkspaceRequest{OrgId: "other", Id: "ws", Name: "Workspace"}); err != nil { t.Fatal(err) }