Drop the bot_user-derived root identity and the empty-identity sentinel. System polls forge connections and owns instance-level work; a polled task stores the polling identity, task_queued and work_request_submitted carry task id, actor and principal, and an empty task identity is refused. Migration 0072 moves empty tasks.identity, the identity-grants root key and the old root's connection permits to System so a re-poll dedups.
215 lines
8.2 KiB
Go
215 lines
8.2 KiB
Go
package daemon
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/domain/identity"
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
|
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/kit"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/kitrun"
|
|
"github.com/samcharles93/archie-core/internal/storage"
|
|
"github.com/samcharles93/archie-core/internal/taskstate"
|
|
)
|
|
|
|
// KitLauncher starts and ends the container of a task whose agent profile
|
|
// is a Kit.
|
|
type KitLauncher interface {
|
|
Launch(ctx context.Context, req kitrun.Request) (*kitrun.Run, error)
|
|
Release(ctx context.Context, run *kitrun.Run) error
|
|
RemoveVolumes(ctx context.Context, volumes []kit.Volume) error
|
|
}
|
|
|
|
// runKitTask runs a task in its Kit's container, every agent stage on the
|
|
// Kit's harness. The worker reaches NATS and the State Store only through
|
|
// the egress relay, and its environment carries no model provider key.
|
|
func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string, profile config.AgentProfile) {
|
|
park := func(reason string, err error) {
|
|
d.Log.ErrorContext(ctx, reason, "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task, reason+": "+err.Error(), taskstate.ParkTransient)
|
|
}
|
|
if d.KitLauncher == nil {
|
|
park("kit profile", fmt.Errorf("kit harness runs are unavailable on this daemon"))
|
|
return
|
|
}
|
|
if err := writeTaskBrief(workDir, task); err != nil {
|
|
park("task.json write failed", err)
|
|
return
|
|
}
|
|
credential, revokeCredential, err := d.runCredential(ctx, task)
|
|
if err != nil {
|
|
park("run credential failed", err)
|
|
return
|
|
}
|
|
defer revokeCredential()
|
|
env, err := kitrun.WorkerEnv(kitrun.Endpoints{
|
|
NATS: d.ConnectedNATS.URL, NATSToken: d.ConnectedNATS.ContainerToken(),
|
|
StateStore: d.ConnectedStateStore.URL, StateStoreToken: credential, StateStoreCA: d.ConnectedStateStore.CA,
|
|
}, os.Getuid(), os.Getgid())
|
|
if err != nil {
|
|
park("kit worker environment", err)
|
|
return
|
|
}
|
|
// The interface is reparsed rather than threaded through pinTaskProfile's
|
|
// return: task.WorkflowDefinitionYAML is already pinned by the time this
|
|
// runs, parsing it is pure and cheap, and pinTaskProfile's signature stays
|
|
// unchanged for its other callers.
|
|
iface, err := workflowtask.ParseWorkflowInterface(task.WorkflowDefinitionYAML)
|
|
if err != nil {
|
|
park("kit workflow interface", err)
|
|
return
|
|
}
|
|
// Org and GrantedServices are the dispatching identity's own facts, read
|
|
// fresh here rather than cached anywhere: the same per-task
|
|
// config.Config d.configFor already resolves for every other identity
|
|
// override (Forge, Budgets, ...).
|
|
taskCfg := d.configFor(task)
|
|
credentialOrg, grantedServices := taskCfg.CredentialAccess(string(identity.SystemID))
|
|
var record *egress.RecordOptions
|
|
if task.IsTestRun() {
|
|
record = &egress.RecordOptions{TaskID: task.ID, ModelHosts: modelHosts(d.Cfg.Get().Providers)}
|
|
}
|
|
run, err := d.KitLauncher.Launch(ctx, kitrun.Request{
|
|
TaskID: task.ID,
|
|
Attempt: task.Attempt,
|
|
Kit: profile.Kit,
|
|
Adapter: profile.Adapter,
|
|
WorkDir: workDir,
|
|
WorkerEnv: env,
|
|
GateRetries: iface.Needs().GateRetries,
|
|
Org: credentialOrg,
|
|
GrantedServices: grantedServices,
|
|
RunCredential: credential,
|
|
Record: record,
|
|
})
|
|
if err != nil {
|
|
park("kit launch failed", err)
|
|
return
|
|
}
|
|
|
|
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
|
|
runCtx, stopWatch := withContainerExit(limitCtx, run.Container)
|
|
d.runViaAgent(runCtx, task, repo, profile, &run.Harness, credential)
|
|
stopWatch()
|
|
stopLimit()
|
|
|
|
if err := d.KitLauncher.Release(ctx, run); err != nil {
|
|
d.Log.WarnContext(ctx, "kit release failed", "task", task.ID, "err", err)
|
|
}
|
|
d.removeEndedKitVolumes(ctx, task.ID, run.Volumes)
|
|
}
|
|
|
|
// removeEndedKitVolumes removes the execution's Kit volumes unless the task
|
|
// parked: a retry of a parked task is the same execution and resumes from
|
|
// them.
|
|
func (d *Daemon) removeEndedKitVolumes(ctx context.Context, taskID int64, volumes []kit.Volume) {
|
|
if len(volumes) == 0 || d.Store == nil {
|
|
return
|
|
}
|
|
lookupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
latest, err := d.Store.TaskByID(lookupCtx, taskID)
|
|
if err != nil || latest == nil || latest.Status == workflow.StatusParked {
|
|
return
|
|
}
|
|
if err := d.KitLauncher.RemoveVolumes(lookupCtx, volumes); err != nil {
|
|
d.Log.WarnContext(ctx, "kit volume removal failed", "task", taskID, "err", err)
|
|
}
|
|
}
|
|
|
|
// ModelEgress opens a native task's model proxy session. The container holds
|
|
// a sentinel under each provider's key variable and reaches the providers
|
|
// through the egress proxy, which swaps in the org's bound key.
|
|
type ModelEgress interface {
|
|
Open(token, org string, providers map[string]config.Provider) (ModelSession, error)
|
|
Close(ctx context.Context, token string)
|
|
}
|
|
|
|
// TaskInstructions answers the instance and organisation instructions for a
|
|
// task's org.
|
|
type TaskInstructions interface {
|
|
TaskInstructions(ctx context.Context, taskID int64) (instance, organisation string, err error)
|
|
}
|
|
|
|
// TaskModels answers what a task's org resolves models against.
|
|
type TaskModels interface {
|
|
TaskModels(ctx context.Context, taskID int64) (config.OrgModels, error)
|
|
}
|
|
|
|
// taskConfig is task's configuration with its org's model aliases and own
|
|
// providers, narrowed to the providers those aliases reach, each carrying the
|
|
// variable its credential is read from. A provider no alias names is not part
|
|
// of the run at all: requiring its credential would park a task whose models
|
|
// never touch it.
|
|
func (d *Daemon) taskConfig(ctx context.Context, task *workflow.Task) (config.Config, error) {
|
|
cfg := d.configFor(task)
|
|
if d.TaskModels != nil {
|
|
models, err := d.TaskModels.TaskModels(ctx, task.ID)
|
|
if err != nil {
|
|
return config.Config{}, fmt.Errorf("resolve the task's models: %w", err)
|
|
}
|
|
cfg = cfg.WithOrgModels(models)
|
|
}
|
|
cfg.Providers = cfg.ModelProviders()
|
|
return cfg, nil
|
|
}
|
|
|
|
// ModelSession is what a native container needs for its model proxy session.
|
|
type ModelSession struct {
|
|
Env []string
|
|
Mounts []storage.Mount
|
|
}
|
|
|
|
// openModelEgress opens task's model session, or refuses: provider keys
|
|
// never reach a container, so without the proxy no model call can run. A
|
|
// workflow with no model step needs no session and no provider credential.
|
|
func (d *Daemon) openModelEgress(ctx context.Context, task *workflow.Task, credential string) (ModelSession, func(), error) {
|
|
if !workflow.UsesModel(task.WorkflowDefinitionYAML) {
|
|
return ModelSession{}, func() {}, nil
|
|
}
|
|
cfg, err := d.taskConfig(ctx, task)
|
|
if err != nil {
|
|
return ModelSession{}, nil, err
|
|
}
|
|
if d.ModelEgress == nil {
|
|
for _, p := range cfg.Providers {
|
|
if p.APIKeyEnv != "" {
|
|
return ModelSession{}, nil, blockedPark{taskstate.BlockerModelCredential, fmt.Errorf("model egress proxy is unavailable on this daemon, and provider keys never enter a container")}
|
|
}
|
|
}
|
|
return ModelSession{}, func() {}, nil
|
|
}
|
|
if credential == "" {
|
|
return ModelSession{}, nil, fmt.Errorf("model egress needs a run credential, and no State Store issues one")
|
|
}
|
|
session, err := d.ModelEgress.Open(credential, task.OrgOrDefault(), cfg.Providers)
|
|
if err != nil {
|
|
return ModelSession{}, nil, blockedPark{taskstate.BlockerModelCredential, err}
|
|
}
|
|
return session, func() { d.ModelEgress.Close(ctx, credential) }, nil
|
|
}
|
|
|
|
// modelHosts is every host a configured provider is reached at. A test run
|
|
// records its deliveries, but a model call is how the run reads, so these hosts
|
|
// are let through whatever their method: this errs broad on purpose, because a
|
|
// provider left out would silently swallow the run's model calls, and no
|
|
// provider host is a delivery destination.
|
|
func modelHosts(providers map[string]config.Provider) []string {
|
|
seen := make(map[string]bool, len(providers))
|
|
hosts := make([]string, 0, len(providers))
|
|
for _, p := range providers {
|
|
host, err := egress.ProviderHost(p.Class, p.BaseURL)
|
|
if err != nil || seen[host] {
|
|
continue
|
|
}
|
|
seen[host] = true
|
|
hosts = append(hosts, host)
|
|
}
|
|
return hosts
|
|
}
|