Files
archie/internal/app/archied/scheduling.go
T

312 lines
11 KiB
Go

package archied
import (
"context"
"errors"
"fmt"
"log/slog"
"strings"
"time"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/domain/access"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/scheduling"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/gateway"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/crondelivery"
"github.com/samcharles93/archie-core/internal/infrastructure/cronevents"
)
// schedulingConfig translates external input into the domain-owned engine
// configuration and connects the selected event infrastructure.
func schedulingConfig(input configuration.SchedulingInput, bus *events.Bus) (scheduling.EngineConfig, error) {
cfg := scheduling.DefaultEngineConfig()
if input.Interval != 0 {
cfg.Interval = input.Interval.Std()
}
if input.MaxParallel != 0 {
cfg.MaxParallel = input.MaxParallel
}
if input.JobTimeout != 0 {
cfg.JobTimeout = input.JobTimeout.Std()
}
switch input.EventsSink {
case "", configuration.EventsSinkNone:
case configuration.EventsSinkBus:
cfg.Events = cronevents.New(bus)
default:
return scheduling.EngineConfig{}, fmt.Errorf("%w: scheduling.events_sink %q (want %q or %q)",
configuration.ErrInvalidInput, input.EventsSink, configuration.EventsSinkBus, configuration.EventsSinkNone)
}
if err := cfg.Validate(); err != nil {
return scheduling.EngineConfig{}, fmt.Errorf("%w: %w", configuration.ErrInvalidInput, err)
}
return cfg, nil
}
// setupScheduling builds the cron/scheduling composition site: a job store,
// a delivery router over the kind runners this composition root can back
// with a real production dependency, and the ticker engine itself. It does
// not start the engine -- startServices does that alongside every other
// subsystem, and the corresponding Stop is registered here via addCleanup so
// shutdown ordering matches every other daemon-lifecycle subsystem.
//
// The "chat" kind delivers through the Messaging Service. The "workflow" kind
// rides gateway.TaskCreator, the path chat's /spawn uses, so it is wired only
// when [chat] identity profiles configure one.
func (b *boot) setupScheduling() error {
cfg, err := schedulingConfig(b.doc.Scheduling, b.bus)
if err != nil {
return fmt.Errorf("scheduling: %w", err)
}
store := scheduleResourceStore{client: b.controlPlane, log: b.log}
if orgs, ok := b.stateStore.(orgDirectory); ok {
store.orgs = orgs
}
chatRunner, err := crondelivery.NewChatCourier(store, func(ctx context.Context, target scheduling.Target, text string) error {
return b.messaging.Deliver(ctx, target.Channel, target.ChatID, text)
})
if err != nil {
return fmt.Errorf("scheduling: build chat runner: %w", err)
}
// Channels are instance-owned, so only the system org may message them.
runners := map[string]scheduling.Runner{scheduling.KindChat: systemOrgOnly{
next: chatRunner, store: store,
refusal: errors.New("chat schedules need an organisation channel; use a workflow schedule"),
}}
if b.chatTasks != nil {
workflowRunner, err := crondelivery.NewWorkflowTask(store, spawnTaskSubmitter{creator: b.chatTasks, identity: b.defaultChatIdentity})
if err != nil {
return fmt.Errorf("scheduling: build workflow runner: %w", err)
}
// The chat identity profiles that submit scheduled work belong to the
// system org; another org's schedule must name the agent it runs as.
runners[scheduling.KindWorkflow] = systemOrgOnly{
next: workflowRunner, store: store,
refusal: errors.New("workflow schedules run only in the system organisation until a schedule names its agent"),
}
}
router, err := crondelivery.NewRouter(store, runners)
if err != nil {
return fmt.Errorf("scheduling: build router: %w", err)
}
engine, err := scheduling.NewEngine(store, router, cfg)
if err != nil {
return fmt.Errorf("scheduling: build engine: %w", err)
}
b.schedulingEngine = engine
b.addCleanup(shutdownSchedulingEngine(engine, b.log))
return nil
}
// orgDirectory finds every organisation and an agent principal to act as in
// each, because the control plane serves a caller only its own org.
type orgDirectory interface {
access.PrincipalSource
ListOrgs(ctx context.Context) ([]org.Org, error)
ListOrgAgents(ctx context.Context, id org.OrgID) ([]org.AgentAssignment, error)
}
// scheduleResourceStore is every organisation's schedules document. Engine job
// ids are "<org>/<job id>" so one engine ticks all orgs, and each lookup or
// run record acts under the owning org's principal.
type scheduleResourceStore struct {
client *controlplane.Client
orgs orgDirectory
log *slog.Logger
}
var (
_ crondelivery.SpecLookup = scheduleResourceStore{}
_ crondelivery.RunRecorder = scheduleResourceStore{}
_ crondelivery.RouterStore = scheduleResourceStore{}
)
func jobKey(owner org.OrgID, id string) string { return string(owner) + "/" + id }
func splitJobKey(key string) (org.OrgID, string) {
owner, id, ok := strings.Cut(key, "/")
if !ok {
return org.DefaultOrgID, key
}
return org.OrgID(owner), id
}
// orgContext returns ctx acting as owner. The system org is the daemon's own,
// so it needs no agent principal.
func (s scheduleResourceStore) orgContext(ctx context.Context, owner org.OrgID) (context.Context, error) {
if owner == org.DefaultOrgID {
return ctx, nil
}
if s.orgs == nil {
return nil, fmt.Errorf("organisation %q: no organisation directory", owner)
}
agents, err := s.orgs.ListOrgAgents(ctx, owner)
if err != nil {
return nil, fmt.Errorf("organisation %q: list agents: %w", owner, err)
}
for _, agent := range agents {
principal, err := s.orgs.PrincipalFor(ctx, agent.IdentityID)
if err == nil && principal.Org == owner {
return org.WithOrg(access.WithPrincipal(ctx, principal), owner), nil
}
}
return nil, fmt.Errorf("organisation %q has no agent to run its schedules as", owner)
}
func (s scheduleResourceStore) orgSchedules(ctx context.Context, key string) (context.Context, string, []scheduling.JobSpec, int64, error) {
owner, id := splitJobKey(key)
octx, err := s.orgContext(ctx, owner)
if err != nil {
return nil, "", nil, 0, err
}
jobs, version, err := s.client.Schedules(octx)
return octx, id, jobs, version, err
}
func (s scheduleResourceStore) Get(ctx context.Context, key string) (scheduling.JobSpec, bool, error) {
_, id, jobs, _, err := s.orgSchedules(ctx, key)
if err != nil {
return scheduling.JobSpec{}, false, err
}
for _, job := range jobs {
if job.ID == id {
return job, true, nil
}
}
return scheduling.JobSpec{}, false, nil
}
// owners lists every org, system org first. Without the directory the system
// org still runs.
func (s scheduleResourceStore) owners(ctx context.Context) []org.OrgID {
owners := []org.OrgID{org.DefaultOrgID}
if s.orgs == nil {
return owners
}
list, err := s.orgs.ListOrgs(ctx)
if err != nil {
s.log.Warn("scheduling: organisations unavailable; running system schedules only", "err", err)
return owners
}
for _, o := range list {
if o.ID != org.DefaultOrgID {
owners = append(owners, o.ID)
}
}
return owners
}
func (s scheduleResourceStore) Due(ctx context.Context, now time.Time) ([]scheduling.Job, error) {
var due []scheduling.Job
for _, owner := range s.owners(ctx) {
octx, err := s.orgContext(ctx, owner)
if err != nil {
s.log.Warn("scheduling: skipping organisation", "org", owner, "err", err)
continue
}
jobs, _, err := s.client.Schedules(octx)
if err != nil {
if owner == org.DefaultOrgID {
return nil, err
}
s.log.Warn("scheduling: organisation schedules unavailable", "org", owner, "err", err)
continue
}
for _, job := range jobs {
if !job.NextRun.After(now) {
due = append(due, scheduling.Job{ID: jobKey(owner, job.ID), Pool: scheduling.Pool(job.Pool), Detail: job.Detail})
}
}
}
return due, nil
}
// systemOrgOnly runs a kind for the system org and refuses it for any other,
// recording the run so the refused job is not retried every tick.
type systemOrgOnly struct {
next scheduling.Runner
store scheduleResourceStore
refusal error
}
func (r systemOrgOnly) Run(ctx context.Context, job scheduling.Job) error {
if owner, _ := splitJobKey(job.ID); owner == org.DefaultOrgID {
return r.next.Run(ctx, job)
}
return errors.Join(fmt.Errorf("schedule %q: %w", job.ID, r.refusal), r.store.MarkRun(ctx, job.ID, time.Now()))
}
func (s scheduleResourceStore) MarkRun(ctx context.Context, key string, runAt time.Time) error {
for range 4 {
octx, id, jobs, version, err := s.orgSchedules(ctx, key)
if err != nil {
return err
}
found := false
for index := range jobs {
if jobs[index].ID != id {
continue
}
found = true
if jobs[index].Schedule.Resolved().Kind == scheduling.ScheduleOnce {
jobs = append(jobs[:index], jobs[index+1:]...)
break
}
next, nextErr := jobs[index].Schedule.NextRun(runAt)
if nextErr != nil {
return nextErr
}
at := runAt.UTC()
jobs[index].LastRun = &at
jobs[index].NextRun = next.UTC()
break
}
if !found {
return fmt.Errorf("%w: id %q", scheduling.ErrJobNotFound, key)
}
_, err = s.client.ReplaceSchedules(octx, jobs, version, "system:scheduler", "scheduler", fmt.Sprintf("schedule-run:%s:%d", key, runAt.UnixNano()))
if err == nil {
return nil
}
if !errors.Is(err, controlplane.ErrVersionConflict) {
return err
}
}
return controlplane.ErrVersionConflict
}
// shutdownSchedulingEngine returns a cleanup that stops the ticker engine.
// The stop path derives its own timeout budget from Background rather than
// the boot context, which is already cancelled by the time shutdown runs --
// the same reasoning as shutdownCuratorRuntime.
func shutdownSchedulingEngine(e *scheduling.Engine, log *slog.Logger) func() {
return func() {
stopCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := e.Stop(stopCtx); err != nil {
log.Error("scheduling engine shutdown", "err", err)
}
}
}
// spawnTaskSubmitter adapts gateway.TaskCreator to crondelivery.TaskSubmitter.
// The job's own id is only a dispatch key to the ticker engine, not an archie
// identity, so it is dropped rather than threaded into the spawn request;
// identity is fixed at construction to the daemon's default chat identity.
type spawnTaskSubmitter struct {
creator gateway.TaskCreator
identity string
}
func (s spawnTaskSubmitter) Submit(ctx context.Context, _, title, body string) error {
_, err := s.creator.CreateTask(ctx, gateway.SpawnRequest{Title: title, Body: body, Identity: s.identity})
return err
}