312 lines
11 KiB
Go
312 lines
11 KiB
Go
package archied
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
|
"github.com/samcharles93/archie-core/internal/domain/access"
|
|
"github.com/samcharles93/archie-core/internal/domain/org"
|
|
"github.com/samcharles93/archie-core/internal/domain/scheduling"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
"github.com/samcharles93/archie-core/internal/gateway"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/crondelivery"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/cronevents"
|
|
)
|
|
|
|
// schedulingConfig translates external input into the domain-owned engine
|
|
// configuration and connects the selected event infrastructure.
|
|
func schedulingConfig(input configuration.SchedulingInput, bus *events.Bus) (scheduling.EngineConfig, error) {
|
|
cfg := scheduling.DefaultEngineConfig()
|
|
if input.Interval != 0 {
|
|
cfg.Interval = input.Interval.Std()
|
|
}
|
|
if input.MaxParallel != 0 {
|
|
cfg.MaxParallel = input.MaxParallel
|
|
}
|
|
if input.JobTimeout != 0 {
|
|
cfg.JobTimeout = input.JobTimeout.Std()
|
|
}
|
|
|
|
switch input.EventsSink {
|
|
case "", configuration.EventsSinkNone:
|
|
case configuration.EventsSinkBus:
|
|
cfg.Events = cronevents.New(bus)
|
|
default:
|
|
return scheduling.EngineConfig{}, fmt.Errorf("%w: scheduling.events_sink %q (want %q or %q)",
|
|
configuration.ErrInvalidInput, input.EventsSink, configuration.EventsSinkBus, configuration.EventsSinkNone)
|
|
}
|
|
if err := cfg.Validate(); err != nil {
|
|
return scheduling.EngineConfig{}, fmt.Errorf("%w: %w", configuration.ErrInvalidInput, err)
|
|
}
|
|
return cfg, nil
|
|
}
|
|
|
|
// setupScheduling builds the cron/scheduling composition site: a job store,
|
|
// a delivery router over the kind runners this composition root can back
|
|
// with a real production dependency, and the ticker engine itself. It does
|
|
// not start the engine -- startServices does that alongside every other
|
|
// subsystem, and the corresponding Stop is registered here via addCleanup so
|
|
// shutdown ordering matches every other daemon-lifecycle subsystem.
|
|
//
|
|
// The "chat" kind delivers through the Messaging Service. The "workflow" kind
|
|
// rides gateway.TaskCreator, the path chat's /spawn uses, so it is wired only
|
|
// when [chat] identity profiles configure one.
|
|
func (b *boot) setupScheduling() error {
|
|
cfg, err := schedulingConfig(b.doc.Scheduling, b.bus)
|
|
if err != nil {
|
|
return fmt.Errorf("scheduling: %w", err)
|
|
}
|
|
|
|
store := scheduleResourceStore{client: b.controlPlane, log: b.log}
|
|
if orgs, ok := b.stateStore.(orgDirectory); ok {
|
|
store.orgs = orgs
|
|
}
|
|
|
|
chatRunner, err := crondelivery.NewChatCourier(store, func(ctx context.Context, target scheduling.Target, text string) error {
|
|
return b.messaging.Deliver(ctx, target.Channel, target.ChatID, text)
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("scheduling: build chat runner: %w", err)
|
|
}
|
|
// Channels are instance-owned, so only the system org may message them.
|
|
runners := map[string]scheduling.Runner{scheduling.KindChat: systemOrgOnly{
|
|
next: chatRunner, store: store,
|
|
refusal: errors.New("chat schedules need an organisation channel; use a workflow schedule"),
|
|
}}
|
|
if b.chatTasks != nil {
|
|
workflowRunner, err := crondelivery.NewWorkflowTask(store, spawnTaskSubmitter{creator: b.chatTasks, identity: b.defaultChatIdentity})
|
|
if err != nil {
|
|
return fmt.Errorf("scheduling: build workflow runner: %w", err)
|
|
}
|
|
// The chat identity profiles that submit scheduled work belong to the
|
|
// system org; another org's schedule must name the agent it runs as.
|
|
runners[scheduling.KindWorkflow] = systemOrgOnly{
|
|
next: workflowRunner, store: store,
|
|
refusal: errors.New("workflow schedules run only in the system organisation until a schedule names its agent"),
|
|
}
|
|
}
|
|
router, err := crondelivery.NewRouter(store, runners)
|
|
if err != nil {
|
|
return fmt.Errorf("scheduling: build router: %w", err)
|
|
}
|
|
engine, err := scheduling.NewEngine(store, router, cfg)
|
|
if err != nil {
|
|
return fmt.Errorf("scheduling: build engine: %w", err)
|
|
}
|
|
|
|
b.schedulingEngine = engine
|
|
b.addCleanup(shutdownSchedulingEngine(engine, b.log))
|
|
return nil
|
|
}
|
|
|
|
// orgDirectory finds every organisation and an agent principal to act as in
|
|
// each, because the control plane serves a caller only its own org.
|
|
type orgDirectory interface {
|
|
access.PrincipalSource
|
|
ListOrgs(ctx context.Context) ([]org.Org, error)
|
|
ListOrgAgents(ctx context.Context, id org.OrgID) ([]org.AgentAssignment, error)
|
|
}
|
|
|
|
// scheduleResourceStore is every organisation's schedules document. Engine job
|
|
// ids are "<org>/<job id>" so one engine ticks all orgs, and each lookup or
|
|
// run record acts under the owning org's principal.
|
|
type scheduleResourceStore struct {
|
|
client *controlplane.Client
|
|
orgs orgDirectory
|
|
log *slog.Logger
|
|
}
|
|
|
|
var (
|
|
_ crondelivery.SpecLookup = scheduleResourceStore{}
|
|
_ crondelivery.RunRecorder = scheduleResourceStore{}
|
|
_ crondelivery.RouterStore = scheduleResourceStore{}
|
|
)
|
|
|
|
func jobKey(owner org.OrgID, id string) string { return string(owner) + "/" + id }
|
|
|
|
func splitJobKey(key string) (org.OrgID, string) {
|
|
owner, id, ok := strings.Cut(key, "/")
|
|
if !ok {
|
|
return org.DefaultOrgID, key
|
|
}
|
|
return org.OrgID(owner), id
|
|
}
|
|
|
|
// orgContext returns ctx acting as owner. The system org is the daemon's own,
|
|
// so it needs no agent principal.
|
|
func (s scheduleResourceStore) orgContext(ctx context.Context, owner org.OrgID) (context.Context, error) {
|
|
if owner == org.DefaultOrgID {
|
|
return ctx, nil
|
|
}
|
|
if s.orgs == nil {
|
|
return nil, fmt.Errorf("organisation %q: no organisation directory", owner)
|
|
}
|
|
agents, err := s.orgs.ListOrgAgents(ctx, owner)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("organisation %q: list agents: %w", owner, err)
|
|
}
|
|
for _, agent := range agents {
|
|
principal, err := s.orgs.PrincipalFor(ctx, agent.IdentityID)
|
|
if err == nil && principal.Org == owner {
|
|
return org.WithOrg(access.WithPrincipal(ctx, principal), owner), nil
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("organisation %q has no agent to run its schedules as", owner)
|
|
}
|
|
|
|
func (s scheduleResourceStore) orgSchedules(ctx context.Context, key string) (context.Context, string, []scheduling.JobSpec, int64, error) {
|
|
owner, id := splitJobKey(key)
|
|
octx, err := s.orgContext(ctx, owner)
|
|
if err != nil {
|
|
return nil, "", nil, 0, err
|
|
}
|
|
jobs, version, err := s.client.Schedules(octx)
|
|
return octx, id, jobs, version, err
|
|
}
|
|
|
|
func (s scheduleResourceStore) Get(ctx context.Context, key string) (scheduling.JobSpec, bool, error) {
|
|
_, id, jobs, _, err := s.orgSchedules(ctx, key)
|
|
if err != nil {
|
|
return scheduling.JobSpec{}, false, err
|
|
}
|
|
for _, job := range jobs {
|
|
if job.ID == id {
|
|
return job, true, nil
|
|
}
|
|
}
|
|
return scheduling.JobSpec{}, false, nil
|
|
}
|
|
|
|
// owners lists every org, system org first. Without the directory the system
|
|
// org still runs.
|
|
func (s scheduleResourceStore) owners(ctx context.Context) []org.OrgID {
|
|
owners := []org.OrgID{org.DefaultOrgID}
|
|
if s.orgs == nil {
|
|
return owners
|
|
}
|
|
list, err := s.orgs.ListOrgs(ctx)
|
|
if err != nil {
|
|
s.log.Warn("scheduling: organisations unavailable; running system schedules only", "err", err)
|
|
return owners
|
|
}
|
|
for _, o := range list {
|
|
if o.ID != org.DefaultOrgID {
|
|
owners = append(owners, o.ID)
|
|
}
|
|
}
|
|
return owners
|
|
}
|
|
|
|
func (s scheduleResourceStore) Due(ctx context.Context, now time.Time) ([]scheduling.Job, error) {
|
|
var due []scheduling.Job
|
|
for _, owner := range s.owners(ctx) {
|
|
octx, err := s.orgContext(ctx, owner)
|
|
if err != nil {
|
|
s.log.Warn("scheduling: skipping organisation", "org", owner, "err", err)
|
|
continue
|
|
}
|
|
jobs, _, err := s.client.Schedules(octx)
|
|
if err != nil {
|
|
if owner == org.DefaultOrgID {
|
|
return nil, err
|
|
}
|
|
s.log.Warn("scheduling: organisation schedules unavailable", "org", owner, "err", err)
|
|
continue
|
|
}
|
|
for _, job := range jobs {
|
|
if !job.NextRun.After(now) {
|
|
due = append(due, scheduling.Job{ID: jobKey(owner, job.ID), Pool: scheduling.Pool(job.Pool), Detail: job.Detail})
|
|
}
|
|
}
|
|
}
|
|
return due, nil
|
|
}
|
|
|
|
// systemOrgOnly runs a kind for the system org and refuses it for any other,
|
|
// recording the run so the refused job is not retried every tick.
|
|
type systemOrgOnly struct {
|
|
next scheduling.Runner
|
|
store scheduleResourceStore
|
|
refusal error
|
|
}
|
|
|
|
func (r systemOrgOnly) Run(ctx context.Context, job scheduling.Job) error {
|
|
if owner, _ := splitJobKey(job.ID); owner == org.DefaultOrgID {
|
|
return r.next.Run(ctx, job)
|
|
}
|
|
return errors.Join(fmt.Errorf("schedule %q: %w", job.ID, r.refusal), r.store.MarkRun(ctx, job.ID, time.Now()))
|
|
}
|
|
|
|
func (s scheduleResourceStore) MarkRun(ctx context.Context, key string, runAt time.Time) error {
|
|
for range 4 {
|
|
octx, id, jobs, version, err := s.orgSchedules(ctx, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
found := false
|
|
for index := range jobs {
|
|
if jobs[index].ID != id {
|
|
continue
|
|
}
|
|
found = true
|
|
if jobs[index].Schedule.Resolved().Kind == scheduling.ScheduleOnce {
|
|
jobs = append(jobs[:index], jobs[index+1:]...)
|
|
break
|
|
}
|
|
next, nextErr := jobs[index].Schedule.NextRun(runAt)
|
|
if nextErr != nil {
|
|
return nextErr
|
|
}
|
|
at := runAt.UTC()
|
|
jobs[index].LastRun = &at
|
|
jobs[index].NextRun = next.UTC()
|
|
break
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("%w: id %q", scheduling.ErrJobNotFound, key)
|
|
}
|
|
_, err = s.client.ReplaceSchedules(octx, jobs, version, "system:scheduler", "scheduler", fmt.Sprintf("schedule-run:%s:%d", key, runAt.UnixNano()))
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if !errors.Is(err, controlplane.ErrVersionConflict) {
|
|
return err
|
|
}
|
|
}
|
|
return controlplane.ErrVersionConflict
|
|
}
|
|
|
|
// shutdownSchedulingEngine returns a cleanup that stops the ticker engine.
|
|
// The stop path derives its own timeout budget from Background rather than
|
|
// the boot context, which is already cancelled by the time shutdown runs --
|
|
// the same reasoning as shutdownCuratorRuntime.
|
|
func shutdownSchedulingEngine(e *scheduling.Engine, log *slog.Logger) func() {
|
|
return func() {
|
|
stopCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := e.Stop(stopCtx); err != nil {
|
|
log.Error("scheduling engine shutdown", "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// spawnTaskSubmitter adapts gateway.TaskCreator to crondelivery.TaskSubmitter.
|
|
// The job's own id is only a dispatch key to the ticker engine, not an archie
|
|
// identity, so it is dropped rather than threaded into the spawn request;
|
|
// identity is fixed at construction to the daemon's default chat identity.
|
|
type spawnTaskSubmitter struct {
|
|
creator gateway.TaskCreator
|
|
identity string
|
|
}
|
|
|
|
func (s spawnTaskSubmitter) Submit(ctx context.Context, _, title, body string) error {
|
|
_, err := s.creator.CreateTask(ctx, gateway.SpawnRequest{Title: title, Body: body, Identity: s.identity})
|
|
return err
|
|
}
|