Task containers log in with their run credential on an external broker and reach only their own subjects, decided by the embedded broker's authenticator. The broker token is never put in a container environment.
376 lines
13 KiB
Go
376 lines
13 KiB
Go
package daemon
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/url"
|
|
"os"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
|
|
"github.com/samcharles93/archie-core/internal/agentexec"
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/container"
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
|
"github.com/samcharles93/archie-core/internal/eventbus"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/egress"
|
|
"github.com/samcharles93/archie-core/internal/storage"
|
|
"github.com/samcharles93/archie-core/internal/taskrun"
|
|
"github.com/samcharles93/archie-core/internal/taskstate"
|
|
)
|
|
|
|
// acquireTaskContainer prepares and acquires the sandbox container for a
|
|
// task, reporting false when it could not -- having already parked the
|
|
// task and logged why. The caller releases the container it returns.
|
|
func (d *Daemon) acquireTaskContainer(
|
|
ctx context.Context,
|
|
task *workflow.Task,
|
|
repo config.Repo,
|
|
workDir string,
|
|
profile config.AgentProfile,
|
|
) (TaskContainer, string, func(), bool) {
|
|
park := func(reason string, err error) {
|
|
d.Log.ErrorContext(ctx, reason, "err", err)
|
|
d.parkBlockedTask(ctx, task, reason+": "+err.Error(), taskstate.ParkTransient, parkBlocker(err))
|
|
}
|
|
|
|
// Resolve the PR number from inputs before the brief is written.
|
|
if task.Workflow == "pr-review" {
|
|
task.PRNumber = task.EffectivePRNumber()
|
|
}
|
|
|
|
if err := writeTaskBrief(workDir, task); err != nil {
|
|
park("task.json write failed", err)
|
|
return nil, "", nil, false
|
|
}
|
|
|
|
// Fetch the pull request with the daemon's forge credential into workDir,
|
|
// so the container reads it without credentials.
|
|
if task.Workflow == "pr-review" {
|
|
if err := prefetchPRReview(ctx, d.forgeFor(ctx, task), task.Owner, task.Repo, task.PRNumber, workDir); err != nil {
|
|
park("pr-review prefetch failed", err)
|
|
return nil, "", nil, false
|
|
}
|
|
}
|
|
|
|
// Guard: Storage may be nil if the daemon was wired incorrectly. In
|
|
// normal operation, Storage is always set when Runtime is set.
|
|
if d.Storage == nil {
|
|
d.Log.ErrorContext(ctx, "storage backend is nil -- cannot acquire container")
|
|
d.parkRunningTask(ctx, task, "storage backend not configured", taskstate.ParkTransient)
|
|
return nil, "", nil, false
|
|
}
|
|
|
|
mounts, err := d.Storage.Setup(ctx, storage.TaskRef{
|
|
WorktreeDir: workDir,
|
|
Ecosystem: repo.Ecosystem,
|
|
PersistentStorage: repo.PersistentStorage,
|
|
Owner: task.Owner,
|
|
Repo: task.Repo,
|
|
})
|
|
if err != nil {
|
|
park("storage setup failed", err)
|
|
return nil, "", nil, false
|
|
}
|
|
|
|
credential, revokeCredential, err := d.runCredential(ctx, task)
|
|
if err != nil {
|
|
d.teardownStorage(ctx, task, repo, workDir)
|
|
park("run credential failed", err)
|
|
return nil, "", nil, false
|
|
}
|
|
|
|
hosts := d.taskEgressHosts(ctx, profile)
|
|
model, closeModel, err := d.openModelEgress(ctx, task, credential, hosts)
|
|
if err != nil {
|
|
revokeCredential()
|
|
d.teardownStorage(ctx, task, repo, workDir)
|
|
park("model egress failed", err)
|
|
return nil, "", nil, false
|
|
}
|
|
revokeRun := revokeCredential
|
|
revokeCredential = func() { closeModel(); revokeRun() }
|
|
|
|
ctr, err := d.Runtime.Acquire(ctx, TaskLaunch{
|
|
Image: profile.Image,
|
|
Mounts: append(mounts, model.Mounts...),
|
|
// The agent may run as root in its container, so its commits land
|
|
// owned by UID 0 on the host. The daemon's UID/GID let it hand the
|
|
// worktree back before the daemon reads it to push.
|
|
Env: append([]string{fmt.Sprintf("WORKTREE_UID=%d", os.Getuid()), fmt.Sprintf("WORKTREE_GID=%d", os.Getgid())}, model.Env...),
|
|
Worker: d.workerEndpoints(credential),
|
|
Providers: model.Providers,
|
|
Hosts: hosts,
|
|
})
|
|
if err != nil {
|
|
revokeCredential()
|
|
// Roll back the storage we just set up: the mounts were created
|
|
// for this task but no container will use them. Leaking them until
|
|
// a later TTL sweep is not acceptable on a backend that allocates
|
|
// real resources (volumes, NFS leases).
|
|
d.teardownStorage(ctx, task, repo, workDir)
|
|
park("container acquire failed", err)
|
|
return nil, "", nil, false
|
|
}
|
|
return ctr, credential, revokeCredential, true
|
|
}
|
|
|
|
// workerEndpoints are NATS and the State Store as the daemon reaches them,
|
|
// with this task's credentials, never the daemon's own. NATS is the endpoint
|
|
// the daemon connected with at startup, not live config: a reloaded [nats]
|
|
// section must not point new containers at a server the daemon is not
|
|
// publishing on.
|
|
func (d *Daemon) workerEndpoints(stateStoreToken string) egress.Endpoints {
|
|
return egress.Endpoints{
|
|
NATS: d.ConnectedNATS.URL,
|
|
StateStore: d.ConnectedStateStore.URL,
|
|
StateStoreToken: stateStoreToken,
|
|
StateStoreCA: d.ConnectedStateStore.CA,
|
|
}
|
|
}
|
|
|
|
// taskEgressHosts are the hosts a task's tools may reach beyond archie: the
|
|
// remote MCP servers its profile allows, and the allowed package registries.
|
|
// An MCP server addressed by IP cannot be named in sandbox policy, so it is
|
|
// left out with a warning rather than failing every task.
|
|
func (d *Daemon) taskEgressHosts(ctx context.Context, profile config.AgentProfile) []string {
|
|
cfg := d.Cfg.Get()
|
|
hosts := slices.Clone(cfg.Containers.EgressHosts)
|
|
for _, server := range cfg.Tools.MCPServers {
|
|
if len(profile.Tools) > 0 && !slices.Contains(profile.Tools, server.Name) {
|
|
continue
|
|
}
|
|
for _, endpoint := range []string{server.URL, server.SSEEndpoint} {
|
|
host, ok := mcpEgressHost(endpoint)
|
|
if !ok {
|
|
if endpoint != "" {
|
|
d.Log.WarnContext(ctx, "mcp server cannot be reached from a sandbox: its url needs a DNS host name", "server", server.Name)
|
|
}
|
|
continue
|
|
}
|
|
if !slices.Contains(hosts, host) {
|
|
hosts = append(hosts, host)
|
|
}
|
|
}
|
|
}
|
|
return hosts
|
|
}
|
|
|
|
// mcpEgressHost is a remote MCP endpoint's host:port, with the scheme's
|
|
// default port when none is given.
|
|
func mcpEgressHost(endpoint string) (string, bool) {
|
|
u, err := url.Parse(strings.TrimSpace(endpoint))
|
|
if err != nil || u.Hostname() == "" || net.ParseIP(u.Hostname()) != nil {
|
|
return "", false
|
|
}
|
|
port := u.Port()
|
|
if port == "" {
|
|
port = "443"
|
|
if u.Scheme == "http" {
|
|
port = "80"
|
|
}
|
|
}
|
|
return net.JoinHostPort(u.Hostname(), port), true
|
|
}
|
|
|
|
// runCredential issues the task's run credential, or "" when no State Store
|
|
// is configured.
|
|
func (d *Daemon) runCredential(ctx context.Context, task *workflow.Task) (string, func(), error) {
|
|
if d.ConnectedStateStore.URL == "" {
|
|
return "", func() {}, nil
|
|
}
|
|
return d.RunCredentials.Issue(ctx, task, d.configFor(task).Budgets.TaskWallClock.Std())
|
|
}
|
|
|
|
// runViaAgent hands a task to archie-agent and waits for its result. It parks
|
|
// the task only when the agent never answered or failed before recording an
|
|
// outcome.
|
|
//
|
|
// credential is the task's run credential; the agent presents it to push the
|
|
// task's branch.
|
|
func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile, credential string) {
|
|
cfg, err := d.taskConfig(ctx, task)
|
|
if err != nil {
|
|
d.parkRunningTask(ctx, task, err.Error(), taskstate.ParkTransient)
|
|
return
|
|
}
|
|
taskCfg := cfg.ForTask()
|
|
d.captureAttemptConfig(ctx, task, taskCfg)
|
|
var instructions workflow.Instructions
|
|
instructions.Instance, instructions.Organisation, err = d.TaskInstructions.TaskInstructions(ctx, task.ID)
|
|
if err != nil {
|
|
d.parkRunningTask(ctx, task, "resolve the task's instructions: "+err.Error(), taskstate.ParkTransient)
|
|
return
|
|
}
|
|
req := taskrun.Request{
|
|
Instructions: instructions,
|
|
Task: task,
|
|
Repo: repo,
|
|
Cfg: taskCfg,
|
|
Providers: agentexec.ProvidersFromConfig(cfg.Providers),
|
|
MCPServers: cfg.Tools.MCPServers,
|
|
RunCredential: credential,
|
|
WorkflowDefinition: task.WorkflowDefinitionYAML,
|
|
Tools: d.toolAllowlist(ctx, task, profile.Tools),
|
|
}
|
|
req.Trace = map[string]string{}
|
|
otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(req.Trace))
|
|
data, err := json.Marshal(req)
|
|
if err != nil {
|
|
d.Log.ErrorContext(ctx, "taskrun encode failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task, "taskrun encode failed: "+err.Error(), taskstate.ParkTransient)
|
|
return
|
|
}
|
|
|
|
reply, err := d.requestTaskRun(ctx, task.ID, data)
|
|
if err != nil {
|
|
if stoppingOnly(ctx) {
|
|
d.Log.InfoContext(ctx, "daemon stopping; task left running for the next daemon to re-queue", "task", task.ID)
|
|
return
|
|
}
|
|
err = runFailure(ctx, err)
|
|
d.Log.ErrorContext(ctx, "taskrun request failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task, "taskrun request failed: "+err.Error(), taskstate.ParkTransient)
|
|
return
|
|
}
|
|
|
|
var resp taskrun.Response
|
|
if err := json.Unmarshal(reply, &resp); err != nil {
|
|
d.Log.ErrorContext(ctx, "taskrun decode response failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task, "taskrun decode response failed: "+err.Error(), taskstate.ParkTransient)
|
|
return
|
|
}
|
|
if resp.Error != "" {
|
|
d.Log.ErrorContext(ctx, "taskrun run failed", "task", task.ID, "err", resp.Error)
|
|
d.parkRunningTask(ctx, task, "taskrun run failed: "+resp.Error, taskstate.ParkTransient)
|
|
return
|
|
}
|
|
// The agent applies the task's own outcome transition, so a status the
|
|
// task cannot reach from running means nothing landed: empty, unknown, or
|
|
// still running. Transient, because the cause is environmental -- a State
|
|
// Store that refused the transition -- and a retry can record one.
|
|
if !taskstate.CanTransition(taskstate.Running, resp.Status) {
|
|
reason := noOutcomeReason(resp.Status)
|
|
d.Log.ErrorContext(ctx, "taskrun run recorded no outcome", "task", task.ID, "status", resp.Status)
|
|
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
|
|
return
|
|
}
|
|
|
|
// Reached only past every failure return above, so this is always a
|
|
// fully decoded, error-free response -- a version is never recorded off
|
|
// a request that never reached an agent, or a response describing a
|
|
// parked/errored task.
|
|
if resp.AgentVersion != "" {
|
|
d.AgentStatus.Observe(resp.AgentVersion, resp.AgentInstallType)
|
|
}
|
|
|
|
d.Log.InfoContext(ctx, "taskrun complete", "task", task.ID, "status", resp.Status)
|
|
}
|
|
|
|
// requestTaskRun publishes the taskrun request, retrying on
|
|
// nats.ErrNoResponders until the container subscribes. Other errors return
|
|
// at once.
|
|
func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) ([]byte, error) {
|
|
subject := agentnats.SubjectForTask(taskID)
|
|
deadline := time.Now().Add(taskRunReadyTimeout)
|
|
backoff := taskRunRetryBackoff
|
|
for {
|
|
reply, err := d.TaskBus.Request(ctx, subject, data)
|
|
if err == nil {
|
|
return reply, nil
|
|
}
|
|
if !errors.Is(err, eventbus.ErrNoResponders) || !time.Now().Before(deadline) {
|
|
return nil, err
|
|
}
|
|
wait := min(backoff, time.Until(deadline))
|
|
if wait <= 0 {
|
|
return nil, err
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
case <-time.After(wait):
|
|
}
|
|
}
|
|
}
|
|
|
|
// captureAttemptConfig records the attempt's effective configuration as an
|
|
// event. Failures are logged.
|
|
func (d *Daemon) captureAttemptConfig(ctx context.Context, task *workflow.Task, cfg config.TaskConfig) {
|
|
document, err := json.Marshal(cfg)
|
|
if err != nil {
|
|
d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err)
|
|
return
|
|
}
|
|
event := events.Event{
|
|
At: time.Now().UTC(),
|
|
Kind: events.KindConfigCaptured,
|
|
TaskID: task.ID,
|
|
Repo: task.Owner + "/" + task.Repo,
|
|
Issue: task.IssueNumber,
|
|
Workflow: task.Workflow,
|
|
Attempt: task.Attempt,
|
|
Data: map[string]any{
|
|
"schema": events.ConfigCapturedSchema,
|
|
// Raw so the document is embedded once, as it was encoded, rather
|
|
// than as a string a reader would have to decode again.
|
|
"document": json.RawMessage(document),
|
|
},
|
|
}
|
|
id, err := d.Store.InsertEvent(ctx, event)
|
|
if err != nil {
|
|
d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err)
|
|
return
|
|
}
|
|
// The assigned ID tells the event sink to broadcast without inserting a
|
|
// duplicate row.
|
|
event.ID = id
|
|
d.emit(ctx, event)
|
|
}
|
|
|
|
// writeTaskBrief writes task.json, the container's boot-time brief.
|
|
func writeTaskBrief(workDir string, task *workflow.Task) error {
|
|
return container.WriteTaskJSON(workDir, container.TaskPayload{
|
|
ID: task.ID, Owner: task.Owner, Repo: task.Repo,
|
|
Number: task.IssueNumber, Title: task.Title, Body: task.Body,
|
|
Labels: strings.Split(task.Labels, ","),
|
|
Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan,
|
|
Inputs: task.Inputs,
|
|
})
|
|
}
|
|
|
|
// noOutcomeReason names why a taskrun reply left the task without an outcome,
|
|
// quoting the status so the log line points at what the agent actually said.
|
|
func noOutcomeReason(status string) string {
|
|
if status == "" {
|
|
return "agent returned no status: the run could not record an outcome"
|
|
}
|
|
return fmt.Sprintf("agent returned unrecognised status %q: the run could not record an outcome", status)
|
|
}
|
|
|
|
const (
|
|
// taskRunReadyTimeout is how long requestTaskRun retries a
|
|
// nats.ErrNoResponders while a new container has not subscribed yet.
|
|
taskRunReadyTimeout = 20 * time.Second
|
|
// taskRunRetryBackoff is the delay between those retries.
|
|
taskRunRetryBackoff = 250 * time.Millisecond
|
|
)
|
|
|
|
// runFailure names why a taskrun request failed: the container's exit or the
|
|
// time limit when either cancelled it, else the request's own error.
|
|
func runFailure(ctx context.Context, err error) error {
|
|
if cause := context.Cause(ctx); errors.Is(cause, errContainerExited) || errors.Is(cause, errTaskTimeLimit) {
|
|
return cause
|
|
}
|
|
return err
|
|
}
|