feat(archied): dispatch every organisation's schedules
This commit is contained in:
1 parent
e579b908f3
commit
c80d4750a8
3 files changed
+159
-27
No files matched your search
@@ -5,9 +5,12 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/domain/access"
|
||||
"github.com/samcharles93/archie-core/internal/domain/org"
|
||||
"github.com/samcharles93/archie-core/internal/domain/scheduling"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/gateway"
|
||||
@@ -60,7 +63,10 @@ func (b *boot) setupScheduling() error {
|
||||
return fmt.Errorf("scheduling: %w", err)
|
||||
}
|
||||
|
||||
store := scheduleResourceStore{client: b.controlPlane}
|
||||
store := scheduleResourceStore{client: b.controlPlane, log: b.log}
|
||||
if orgs, ok := b.stateStore.(orgDirectory); ok {
|
||||
store.orgs = orgs
|
||||
}
|
||||
|
||||
chatRunner, err := crondelivery.NewChatCourier(store, func(ctx context.Context, target scheduling.Target, text string) error {
|
||||
return b.messaging.Deliver(ctx, target.Channel, target.ChatID, text)
|
||||
@@ -68,13 +74,22 @@ func (b *boot) setupScheduling() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("scheduling: build chat runner: %w", err)
|
||||
}
|
||||
runners := map[string]scheduling.Runner{scheduling.KindChat: chatRunner}
|
||||
// Channels are instance-owned, so only the system org may message them.
|
||||
runners := map[string]scheduling.Runner{scheduling.KindChat: systemOrgOnly{
|
||||
next: chatRunner, store: store,
|
||||
refusal: errors.New("chat schedules need an organisation channel; use a workflow schedule"),
|
||||
}}
|
||||
if b.chatTasks != nil {
|
||||
workflowRunner, err := crondelivery.NewWorkflowTask(store, spawnTaskSubmitter{creator: b.chatTasks, identity: b.defaultChatIdentity})
|
||||
if err != nil {
|
||||
return fmt.Errorf("scheduling: build workflow runner: %w", err)
|
||||
}
|
||||
runners[scheduling.KindWorkflow] = workflowRunner
|
||||
// The chat identity profiles that submit scheduled work belong to the
|
||||
// system org; another org's schedule must name the agent it runs as.
|
||||
runners[scheduling.KindWorkflow] = systemOrgOnly{
|
||||
next: workflowRunner, store: store,
|
||||
refusal: errors.New("workflow schedules run only in the system organisation until a schedule names its agent"),
|
||||
}
|
||||
}
|
||||
router, err := crondelivery.NewRouter(store, runners)
|
||||
if err != nil {
|
||||
@@ -90,22 +105,73 @@ func (b *boot) setupScheduling() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// orgDirectory finds every organisation and an agent principal to act as in
|
||||
// each, because the control plane serves a caller only its own org.
|
||||
type orgDirectory interface {
|
||||
access.PrincipalSource
|
||||
ListOrgs(ctx context.Context) ([]org.Org, error)
|
||||
ListOrgAgents(ctx context.Context, id org.OrgID) ([]org.AgentAssignment, error)
|
||||
}
|
||||
|
||||
// scheduleResourceStore is every organisation's schedules document. Engine job
|
||||
// ids are "<org>/<job id>" so one engine ticks all orgs, and each lookup or
|
||||
// run record acts under the owning org's principal.
|
||||
type scheduleResourceStore struct {
|
||||
client *controlplane.Client
|
||||
orgs orgDirectory
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// The schedules document behind the control plane is the production
|
||||
// RouterStore: it resolves a job id to its spec and records a completed run.
|
||||
// The assertions this replaces used to pin the retired file-backed store;
|
||||
// crondelivery's contract comments name this type as the implementation.
|
||||
var (
|
||||
_ crondelivery.SpecLookup = scheduleResourceStore{}
|
||||
_ crondelivery.RunRecorder = scheduleResourceStore{}
|
||||
_ crondelivery.RouterStore = scheduleResourceStore{}
|
||||
)
|
||||
|
||||
func (s scheduleResourceStore) Get(ctx context.Context, id string) (scheduling.JobSpec, bool, error) {
|
||||
jobs, _, err := s.client.Schedules(ctx)
|
||||
func jobKey(owner org.OrgID, id string) string { return string(owner) + "/" + id }
|
||||
|
||||
func splitJobKey(key string) (org.OrgID, string) {
|
||||
owner, id, ok := strings.Cut(key, "/")
|
||||
if !ok {
|
||||
return org.DefaultOrgID, key
|
||||
}
|
||||
return org.OrgID(owner), id
|
||||
}
|
||||
|
||||
// orgContext returns ctx acting as owner. The system org is the daemon's own,
|
||||
// so it needs no agent principal.
|
||||
func (s scheduleResourceStore) orgContext(ctx context.Context, owner org.OrgID) (context.Context, error) {
|
||||
if owner == org.DefaultOrgID {
|
||||
return ctx, nil
|
||||
}
|
||||
if s.orgs == nil {
|
||||
return nil, fmt.Errorf("organisation %q: no organisation directory", owner)
|
||||
}
|
||||
agents, err := s.orgs.ListOrgAgents(ctx, owner)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("organisation %q: list agents: %w", owner, err)
|
||||
}
|
||||
for _, agent := range agents {
|
||||
principal, err := s.orgs.PrincipalFor(ctx, agent.IdentityID)
|
||||
if err == nil && principal.Org == owner {
|
||||
return org.WithOrg(access.WithPrincipal(ctx, principal), owner), nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("organisation %q has no agent to run its schedules as", owner)
|
||||
}
|
||||
|
||||
func (s scheduleResourceStore) orgSchedules(ctx context.Context, key string) (context.Context, string, []scheduling.JobSpec, int64, error) {
|
||||
owner, id := splitJobKey(key)
|
||||
octx, err := s.orgContext(ctx, owner)
|
||||
if err != nil {
|
||||
return nil, "", nil, 0, err
|
||||
}
|
||||
jobs, version, err := s.client.Schedules(octx)
|
||||
return octx, id, jobs, version, err
|
||||
}
|
||||
|
||||
func (s scheduleResourceStore) Get(ctx context.Context, key string) (scheduling.JobSpec, bool, error) {
|
||||
_, id, jobs, _, err := s.orgSchedules(ctx, key)
|
||||
if err != nil {
|
||||
return scheduling.JobSpec{}, false, err
|
||||
}
|
||||
@@ -117,23 +183,69 @@ func (s scheduleResourceStore) Get(ctx context.Context, id string) (scheduling.J
|
||||
return scheduling.JobSpec{}, false, nil
|
||||
}
|
||||
|
||||
func (s scheduleResourceStore) Due(ctx context.Context, now time.Time) ([]scheduling.Job, error) {
|
||||
jobs, _, err := s.client.Schedules(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// owners lists every org, system org first. Without the directory the system
|
||||
// org still runs.
|
||||
func (s scheduleResourceStore) owners(ctx context.Context) []org.OrgID {
|
||||
owners := []org.OrgID{org.DefaultOrgID}
|
||||
if s.orgs == nil {
|
||||
return owners
|
||||
}
|
||||
due := make([]scheduling.Job, 0, len(jobs))
|
||||
for _, job := range jobs {
|
||||
if !job.NextRun.After(now) {
|
||||
due = append(due, scheduling.Job{ID: job.ID, Pool: scheduling.Pool(job.Pool), Detail: job.Detail})
|
||||
list, err := s.orgs.ListOrgs(ctx)
|
||||
if err != nil {
|
||||
s.log.Warn("scheduling: organisations unavailable; running system schedules only", "err", err)
|
||||
return owners
|
||||
}
|
||||
for _, o := range list {
|
||||
if o.ID != org.DefaultOrgID {
|
||||
owners = append(owners, o.ID)
|
||||
}
|
||||
}
|
||||
return owners
|
||||
}
|
||||
|
||||
func (s scheduleResourceStore) Due(ctx context.Context, now time.Time) ([]scheduling.Job, error) {
|
||||
var due []scheduling.Job
|
||||
for _, owner := range s.owners(ctx) {
|
||||
octx, err := s.orgContext(ctx, owner)
|
||||
if err != nil {
|
||||
s.log.Warn("scheduling: skipping organisation", "org", owner, "err", err)
|
||||
continue
|
||||
}
|
||||
jobs, _, err := s.client.Schedules(octx)
|
||||
if err != nil {
|
||||
if owner == org.DefaultOrgID {
|
||||
return nil, err
|
||||
}
|
||||
s.log.Warn("scheduling: organisation schedules unavailable", "org", owner, "err", err)
|
||||
continue
|
||||
}
|
||||
for _, job := range jobs {
|
||||
if !job.NextRun.After(now) {
|
||||
due = append(due, scheduling.Job{ID: jobKey(owner, job.ID), Pool: scheduling.Pool(job.Pool), Detail: job.Detail})
|
||||
}
|
||||
}
|
||||
}
|
||||
return due, nil
|
||||
}
|
||||
|
||||
func (s scheduleResourceStore) MarkRun(ctx context.Context, id string, runAt time.Time) error {
|
||||
// systemOrgOnly runs a kind for the system org and refuses it for any other,
|
||||
// recording the run so the refused job is not retried every tick.
|
||||
type systemOrgOnly struct {
|
||||
next scheduling.Runner
|
||||
store scheduleResourceStore
|
||||
refusal error
|
||||
}
|
||||
|
||||
func (r systemOrgOnly) Run(ctx context.Context, job scheduling.Job) error {
|
||||
if owner, _ := splitJobKey(job.ID); owner == org.DefaultOrgID {
|
||||
return r.next.Run(ctx, job)
|
||||
}
|
||||
return errors.Join(fmt.Errorf("schedule %q: %w", job.ID, r.refusal), r.store.MarkRun(ctx, job.ID, time.Now()))
|
||||
}
|
||||
|
||||
func (s scheduleResourceStore) MarkRun(ctx context.Context, key string, runAt time.Time) error {
|
||||
for range 4 {
|
||||
jobs, version, err := s.client.Schedules(ctx)
|
||||
octx, id, jobs, version, err := s.orgSchedules(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -157,9 +269,9 @@ func (s scheduleResourceStore) MarkRun(ctx context.Context, id string, runAt tim
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("%w: id %q", scheduling.ErrJobNotFound, id)
|
||||
return fmt.Errorf("%w: id %q", scheduling.ErrJobNotFound, key)
|
||||
}
|
||||
_, err = s.client.ReplaceSchedules(ctx, jobs, version, "system:scheduler", "scheduler", fmt.Sprintf("schedule-run:%s:%d", id, runAt.UnixNano()))
|
||||
_, err = s.client.ReplaceSchedules(octx, jobs, version, "system:scheduler", "scheduler", fmt.Sprintf("schedule-run:%s:%d", key, runAt.UnixNano()))
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -71,8 +71,9 @@ func (s *server) ListOrgs(ctx context.Context, _ *pb.ListOrgsRequest) (*pb.ListO
|
||||
if err != nil {
|
||||
return nil, s.orgErr("ListOrgs", err)
|
||||
}
|
||||
// Only an instance admin sees every org; anyone else sees their own.
|
||||
if p, ok := access.PrincipalFromContext(ctx); !ok || !p.InstanceAdmin() {
|
||||
// Only an instance admin or an internal service sees every org; anyone
|
||||
// else sees their own.
|
||||
if !readsEveryOrg(ctx) {
|
||||
caller := org.OrgFromContext(ctx)
|
||||
values = slices.DeleteFunc(values, func(o org.Org) bool { return o.ID != caller })
|
||||
}
|
||||
@@ -83,8 +84,17 @@ func (s *server) ListOrgs(ctx context.Context, _ *pb.ListOrgsRequest) (*pb.ListO
|
||||
return &pb.ListOrgsResponse{Orgs: out}, nil
|
||||
}
|
||||
|
||||
// orgInForce is the org a request is served for: the one the caller's
|
||||
// principal carries. A request field cannot choose another tenant.
|
||||
// readsEveryOrg reports whether ctx may read another org's directory: an
|
||||
// instance admin, or an internal service acting for no one and confined to no
|
||||
// org, such as the daemon ticking every org's schedules.
|
||||
func readsEveryOrg(ctx context.Context) bool {
|
||||
if p, ok := access.PrincipalFromContext(ctx); ok {
|
||||
return p.InstanceAdmin()
|
||||
}
|
||||
_, scoped := org.Scope(ctx)
|
||||
return !scoped
|
||||
}
|
||||
|
||||
// orgFor is the org a request acts in: the caller's own, or the one it names
|
||||
// when the caller is an instance admin. Anyone else naming another org still
|
||||
// acts in their own, so the contract cannot reach another tenant.
|
||||
@@ -221,7 +231,11 @@ func (s *server) ListOrgAgents(ctx context.Context, r *pb.ListOrgAgentsRequest)
|
||||
if s.deps.Orgs == nil {
|
||||
return nil, errOrgsUnavailable
|
||||
}
|
||||
values, err := s.deps.Orgs.ListOrgAgents(ctx, orgFor(ctx, r.GetOrgId()))
|
||||
owner := orgFor(ctx, r.GetOrgId())
|
||||
if r.GetOrgId() != "" && readsEveryOrg(ctx) {
|
||||
owner = org.OrgID(r.GetOrgId())
|
||||
}
|
||||
values, err := s.deps.Orgs.ListOrgAgents(ctx, owner)
|
||||
if err != nil {
|
||||
return nil, s.orgErr("ListOrgAgents", err)
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@ func (*fakeOrgs) OrgForIdentity(context.Context, identity.IdentityID) (org.OrgID
|
||||
func TestOrgRPCsActInTheCallersOrg(t *testing.T) {
|
||||
repo := &fakeOrgs{}
|
||||
s := &server{deps: Deps{Orgs: repo, Log: slog.New(slog.DiscardHandler)}}
|
||||
ctx := org.WithOrg(context.Background(), "acme")
|
||||
ctx := org.WithScope(org.WithOrg(context.Background(), "acme"), "acme")
|
||||
|
||||
got, err := s.GetOrg(ctx, &pb.GetOrgRequest{OrgId: "other"})
|
||||
if err != nil || got.GetOrg().GetId() != "acme" {
|
||||
@@ -97,6 +97,12 @@ func TestOrgRPCsActInTheCallersOrg(t *testing.T) {
|
||||
if err != nil || len(agents.GetAgents()) != 1 || agents.Agents[0].OrgId != "acme" {
|
||||
t.Fatalf("assignment read escaped org: %v %v", agents, err)
|
||||
}
|
||||
// An unscoped internal service, the daemon ticking every org's
|
||||
// schedules, reads the org it names.
|
||||
agents, err = s.ListOrgAgents(context.Background(), &pb.ListOrgAgentsRequest{OrgId: "acme"})
|
||||
if err != nil || len(agents.GetAgents()) != 1 {
|
||||
t.Fatalf("internal service could not read a named org: %v %v", agents, err)
|
||||
}
|
||||
if _, err := s.CreateWorkspace(ctx, &pb.CreateWorkspaceRequest{OrgId: "other", Id: "ws", Name: "Workspace"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user