442 lines
16 KiB
Go
442 lines
16 KiB
Go
// Package storage defines the pluggable storage backend interface for agent
|
|
// container execution. The Docker backend (DockerBackend) is the MVP
|
|
// implementation -- no NFS/SMB/S3 yet, just Docker volumes + bind mounts.
|
|
//
|
|
// /data/ volume layout (implemented):
|
|
//
|
|
// /data/
|
|
// worktree/ -- bind mount of host worktree
|
|
// .git/task.json -- boot-time brief (WriteTaskJSON). Under .git so
|
|
// the agent's own commit cannot sweep it onto the
|
|
// task branch: go-git's Add ignores .gitignore.
|
|
// repo/ -- per-repo persistent volume (optional, on request)
|
|
// session.jsonl -- agent session transcript, appended per stage
|
|
// memory.jsonl -- cross-session project memory
|
|
// plugins/ -- daemon-staged bundled plugins
|
|
// cache/
|
|
// go/ -- GOMODCACHE (shared across all tasks)
|
|
// node/ -- npm cache (shared)
|
|
// pnpm/ -- pnpm store (shared)
|
|
// deno/ -- Deno module cache (shared)
|
|
// bun/ -- Bun package cache (shared)
|
|
// pip/ -- pip cache (shared)
|
|
// cargo/ -- Rust cargo cache (shared)
|
|
// mcp-npm/ -- npm cache for npx-launched per-task MCP
|
|
// servers (shared, mounted regardless of the
|
|
// task repo's ecosystem -- see MCPNPMCacheMountDir)
|
|
//
|
|
// Cache volumes are named Docker volumes created once and shared across all
|
|
// tasks. They are never deleted -- the daemon operator manages them.
|
|
package storage
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/moby/moby/api/types/mount"
|
|
"github.com/moby/moby/client"
|
|
)
|
|
|
|
// ── types ──────────────────────────────────────────────────────────────
|
|
|
|
// TaskRef identifies a task for storage setup.
|
|
type TaskRef struct {
|
|
Owner, Repo string
|
|
IssueNumber int
|
|
Ecosystem string // "go", "node", "python", "rust", "custom"
|
|
WorktreeDir string // host path to the git worktree
|
|
PersistentStorage bool // if true, attach a per-repo Docker volume at /data/repo
|
|
}
|
|
|
|
// Mount describes a container mount point.
|
|
type Mount struct {
|
|
Type string // "bind" or "volume"
|
|
Source string // host path (bind) or volume name (volume)
|
|
Destination string // container path
|
|
}
|
|
|
|
// Mount type constants.
|
|
const (
|
|
MountTypeBind = "bind"
|
|
MountTypeVolume = "volume"
|
|
)
|
|
|
|
// AppendJSONLine appends one complete JSON value to a JSONL file. The value is
|
|
// marshalled before opening the file so a marshal failure cannot leave a
|
|
// partial record behind.
|
|
func AppendJSONLine(path string, value any) error {
|
|
record, err := json.Marshal(value)
|
|
if err != nil {
|
|
return fmt.Errorf("marshal JSONL record: %w", err)
|
|
}
|
|
record = append(record, '\n')
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
return fmt.Errorf("create JSONL directory: %w", err)
|
|
}
|
|
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
|
|
if err != nil {
|
|
return fmt.Errorf("open JSONL file: %w", err)
|
|
}
|
|
if _, err := f.Write(record); err != nil {
|
|
_ = f.Close()
|
|
return fmt.Errorf("append JSONL record: %w", err)
|
|
}
|
|
if err := f.Close(); err != nil {
|
|
return fmt.Errorf("close JSONL file: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ReadJSONLines decodes each line of the JSONL file at path as a T, calling
|
|
// fn for each. A path that does not exist yet is not an error -- no log
|
|
// written and an empty log mean the same thing to a caller -- so fn is
|
|
// simply never called. A malformed line, or an error fn itself returns,
|
|
// stops the scan and is returned wrapped.
|
|
//
|
|
// This is AppendJSONLine's read-side counterpart, kept in the package that
|
|
// owns the format: see organisation.md's "a package owns its own format end
|
|
// to end."
|
|
func ReadJSONLines[T any](path string, fn func(T) error) error {
|
|
f, err := os.Open(path)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("open JSONL file: %w", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
scanner.Buffer(make([]byte, 64*1024), 8*1024*1024)
|
|
for scanner.Scan() {
|
|
var record T
|
|
if err := json.Unmarshal(scanner.Bytes(), &record); err != nil {
|
|
return fmt.Errorf("decode JSONL record: %w", err)
|
|
}
|
|
if err := fn(record); err != nil {
|
|
return fmt.Errorf("handle JSONL record: %w", err)
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return fmt.Errorf("scan JSONL file: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// StagePlugin writes a bundled plugin unless persistent project storage
|
|
// already contains a plugin with the same filename. Persistent copies are
|
|
// operator-owned overrides and therefore win over daemon-staged source.
|
|
func StagePlugin(dir, name string, source []byte) error {
|
|
if filepath.Base(name) != name || name == "." || name == "" {
|
|
return fmt.Errorf("invalid plugin filename %q", name)
|
|
}
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
return fmt.Errorf("create plugin directory: %w", err)
|
|
}
|
|
path := filepath.Join(dir, name)
|
|
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
|
if errors.Is(err, os.ErrExist) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("create staged plugin: %w", err)
|
|
}
|
|
if _, err := f.Write(source); err != nil {
|
|
_ = f.Close()
|
|
return fmt.Errorf("write staged plugin: %w", err)
|
|
}
|
|
if err := f.Close(); err != nil {
|
|
return fmt.Errorf("close staged plugin: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// WorktreeMountDir is the fixed container path where a task's worktree is
|
|
// bind-mounted. archie-agent uses this path directly rather than the host
|
|
// path archied's worktree.Manager reports -- the two processes see the same
|
|
// files at different paths.
|
|
const WorktreeMountDir = "/data/worktree"
|
|
|
|
// Persistent storage paths share the per-repository volume so session output,
|
|
// project memory, and staged plugins survive individual task containers.
|
|
const (
|
|
PersistentMountDir = "/data/repo"
|
|
SessionPath = PersistentMountDir + "/session.jsonl"
|
|
MemoryPath = PersistentMountDir + "/memory.jsonl"
|
|
PluginsDir = PersistentMountDir + "/plugins"
|
|
)
|
|
|
|
// MCPNPMCacheMountDir is the fixed container path where the npm package
|
|
// cache for npx-launched per-task MCP servers is mounted. Unlike the
|
|
// ecosystem caches below, this is mounted on every task container
|
|
// regardless of the task repo's language: per-task MCP servers come from
|
|
// daemon-wide config (config.MCPServer), not from anything specific to the
|
|
// checked-out repo, so an npx-based server needs the cache warm whether the
|
|
// task is a Go repo, a Python repo, or anything else.
|
|
const MCPNPMCacheMountDir = "/data/cache/mcp-npm"
|
|
|
|
// Backend prepares container storage. Each container runtime backend
|
|
// (Docker, future containerd, etc.) implements this interface.
|
|
type Backend interface {
|
|
// Setup returns the mount specs for a task container. Must include
|
|
// at minimum a bind mount for the worktree at /data/worktree.
|
|
Setup(ctx context.Context, task TaskRef) ([]Mount, error)
|
|
// Teardown cleans up ephemeral storage. Cache volumes survive.
|
|
Teardown(ctx context.Context, task TaskRef) error
|
|
// CleanupExpired removes persistent per-repo storage older than ttl.
|
|
// Shared ecosystem caches and storage not owned by Archie survive.
|
|
CleanupExpired(ctx context.Context, ttl time.Duration) (int, error)
|
|
}
|
|
|
|
// ── cache mounts per ecosystem ─────────────────────────────────────────
|
|
|
|
// cacheVolume describes a named Docker volume that is shared across tasks.
|
|
type cacheVolume struct {
|
|
Name string // Docker volume name
|
|
MountPath string // container destination path
|
|
}
|
|
|
|
// alwaysOnCacheVolumes are shared cache volumes mounted on every task
|
|
// container regardless of the task repo's ecosystem, because what they
|
|
// cache isn't tied to the repo being worked on. The npx package cache for
|
|
// per-task MCP servers is the only current instance: MCP servers are
|
|
// daemon-wide config, not per-repo, so an npx-based one needs a warm cache
|
|
// no matter which repo the task happens to be against.
|
|
var alwaysOnCacheVolumes = []cacheVolume{
|
|
{Name: "archie-cache-mcp-npm", MountPath: MCPNPMCacheMountDir},
|
|
}
|
|
|
|
// cacheVolumesByEcosystem maps ecosystem names to their shared cache volumes.
|
|
// Volumes are created once by the pool and never deleted.
|
|
var cacheVolumesByEcosystem = map[string][]cacheVolume{
|
|
"go": {
|
|
{Name: "archie-cache-go", MountPath: "/data/cache/go"},
|
|
},
|
|
"node": {
|
|
{Name: "archie-cache-node", MountPath: "/data/cache/node"},
|
|
{Name: "archie-cache-pnpm", MountPath: "/data/cache/pnpm"},
|
|
},
|
|
"typescript": {
|
|
{Name: "archie-cache-node", MountPath: "/data/cache/node"},
|
|
{Name: "archie-cache-pnpm", MountPath: "/data/cache/pnpm"},
|
|
},
|
|
"deno": {
|
|
{Name: "archie-cache-deno", MountPath: "/data/cache/deno"},
|
|
},
|
|
"bun": {
|
|
{Name: "archie-cache-bun", MountPath: "/data/cache/bun"},
|
|
},
|
|
"python": {
|
|
{Name: "archie-cache-pip", MountPath: "/data/cache/pip"},
|
|
},
|
|
"rust": {
|
|
{Name: "archie-cache-cargo", MountPath: "/data/cache/cargo"},
|
|
},
|
|
}
|
|
|
|
// cacheMounts returns the cache mount specs for an ecosystem -- the
|
|
// always-on volumes plus any matched by ecosystem -- sorted by destination
|
|
// path for deterministic container configuration. Ecosystem is
|
|
// case-insensitive -- "Python" and "python" are equivalent. An unknown or
|
|
// empty ecosystem still returns the always-on volumes.
|
|
func cacheMounts(ecosystem string) []Mount {
|
|
vols := append([]cacheVolume(nil), alwaysOnCacheVolumes...)
|
|
vols = append(vols, cacheVolumesByEcosystem[strings.ToLower(ecosystem)]...)
|
|
|
|
mounts := make([]Mount, len(vols))
|
|
for i, v := range vols {
|
|
mounts[i] = Mount{
|
|
Type: MountTypeVolume,
|
|
Source: v.Name,
|
|
Destination: v.MountPath,
|
|
}
|
|
}
|
|
sort.Slice(mounts, func(i, j int) bool {
|
|
return mounts[i].Destination < mounts[j].Destination
|
|
})
|
|
return mounts
|
|
}
|
|
|
|
// ── error types ────────────────────────────────────────────────────────
|
|
|
|
// ErrVolumeCreate is returned when a Docker volume cannot be created.
|
|
type ErrVolumeCreate struct {
|
|
Volume string
|
|
Cause error
|
|
}
|
|
|
|
func (e *ErrVolumeCreate) Error() string {
|
|
return "create volume " + e.Volume + ": " + e.Cause.Error()
|
|
}
|
|
|
|
func (e *ErrVolumeCreate) Unwrap() error { return e.Cause }
|
|
|
|
// ── Docker backend ─────────────────────────────────────────────────────
|
|
|
|
// DockerBackend implements Backend for Docker via the moby client.
|
|
type DockerBackend struct {
|
|
cli *client.Client
|
|
now func() time.Time
|
|
}
|
|
|
|
const (
|
|
storageLabel = "com.samcharles93.archie.storage"
|
|
storageKindRepo = "repo"
|
|
storageKindCache = "cache"
|
|
)
|
|
|
|
// NewDockerBackend creates a Docker storage backend.
|
|
func NewDockerBackend(cli *client.Client) *DockerBackend {
|
|
return &DockerBackend{cli: cli}
|
|
}
|
|
|
|
// ensureVolume creates a Docker volume if it doesn't already exist.
|
|
// It is idempotent: concurrent callers racing to create the same volume
|
|
// will not see an error -- "already exists" is treated as success.
|
|
//
|
|
// When d.cli is nil (unit tests), ensureVolume returns nil without
|
|
// creating a volume. The returned mounts will fail at container create
|
|
// time if Docker cannot find the volume. Production code always sets
|
|
// cli via NewDockerBackend.
|
|
func (d *DockerBackend) ensureVolume(ctx context.Context, name string, labels map[string]string) error {
|
|
if d.cli == nil {
|
|
return nil // nil client for testing -- volumes must be pre-created
|
|
}
|
|
// VolumeInspectOptions{} is the empty struct form (requires moby v25+).
|
|
// The go.mod pins a compatible version; zero value is always valid.
|
|
_, err := d.cli.VolumeInspect(ctx, name, client.VolumeInspectOptions{})
|
|
if err == nil {
|
|
return nil // already exists
|
|
}
|
|
_, err = d.cli.VolumeCreate(ctx, client.VolumeCreateOptions{
|
|
Name: name,
|
|
Driver: "local",
|
|
Labels: labels,
|
|
})
|
|
if err != nil {
|
|
// TOCTOU: a concurrent caller may have created the volume between
|
|
// our inspect and create calls. Docker returns an error for
|
|
// duplicate volumes -- treat that as success.
|
|
if strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "already in use") {
|
|
return nil
|
|
}
|
|
return &ErrVolumeCreate{Volume: name, Cause: err}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// repoVolumeName returns the Docker volume name for a repo's persistent storage.
|
|
func repoVolumeName(owner, repo string) string {
|
|
return fmt.Sprintf("archie-repo-%s-%s", owner, repo)
|
|
}
|
|
|
|
// Setup returns the mount list for a task container. Mounts are ordered:
|
|
// 1. /data/worktree bind mount (always first)
|
|
// 2. /data/repo volume mount (only when PersistentStorage is true)
|
|
// 3. cache volume mounts: the always-on volumes plus /data/cache/<ecosystem>
|
|
// (sorted together by destination)
|
|
func (d *DockerBackend) Setup(ctx context.Context, task TaskRef) ([]Mount, error) {
|
|
mounts := []Mount{
|
|
{
|
|
Type: MountTypeBind,
|
|
Source: task.WorktreeDir,
|
|
Destination: WorktreeMountDir,
|
|
},
|
|
}
|
|
|
|
// Per-repo persistent volume.
|
|
if task.PersistentStorage && task.Owner != "" && task.Repo != "" {
|
|
volName := repoVolumeName(task.Owner, task.Repo)
|
|
if err := d.ensureVolume(ctx, volName, map[string]string{storageLabel: storageKindRepo}); err != nil {
|
|
return nil, err
|
|
}
|
|
mounts = append(mounts, Mount{
|
|
Type: MountTypeVolume,
|
|
Source: volName,
|
|
Destination: PersistentMountDir,
|
|
})
|
|
}
|
|
|
|
caches := cacheMounts(task.Ecosystem)
|
|
for _, m := range caches {
|
|
if err := d.ensureVolume(ctx, m.Source, map[string]string{storageLabel: storageKindCache}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
mounts = append(mounts, caches...)
|
|
|
|
return mounts, nil
|
|
}
|
|
|
|
// Teardown is a no-op for Docker backend. Cache volumes survive task
|
|
// completion so subsequent tasks benefit from warmed caches.
|
|
func (d *DockerBackend) Teardown(ctx context.Context, task TaskRef) error {
|
|
return nil
|
|
}
|
|
|
|
// CleanupExpired removes Archie-owned per-repo volumes whose creation time is
|
|
// at least ttl old. Docker volume labels are immutable, so the TTL is a hard
|
|
// maximum age rather than an idle lease. Removal is non-forced: a volume still
|
|
// attached to a running container is retained and retried on the next sweep.
|
|
func (d *DockerBackend) CleanupExpired(ctx context.Context, ttl time.Duration) (int, error) {
|
|
if d.cli == nil || ttl <= 0 {
|
|
return 0, nil
|
|
}
|
|
result, err := d.cli.VolumeList(ctx, client.VolumeListOptions{
|
|
Filters: client.Filters{}.Add("label", storageLabel+"="+storageKindRepo),
|
|
})
|
|
if err != nil {
|
|
return 0, fmt.Errorf("list persistent volumes: %w", err)
|
|
}
|
|
|
|
now := time.Now()
|
|
if d.now != nil {
|
|
now = d.now()
|
|
}
|
|
var (
|
|
removed int
|
|
errs []error
|
|
)
|
|
for _, vol := range result.Items {
|
|
// Defend against Docker implementations that ignore label filters.
|
|
if vol.Labels[storageLabel] != storageKindRepo {
|
|
continue
|
|
}
|
|
created, err := time.Parse(time.RFC3339Nano, vol.CreatedAt)
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("persistent volume %s created_at %q: %w", vol.Name, vol.CreatedAt, err))
|
|
continue
|
|
}
|
|
if now.Sub(created) < ttl {
|
|
continue
|
|
}
|
|
if _, err := d.cli.VolumeRemove(ctx, vol.Name, client.VolumeRemoveOptions{}); err != nil {
|
|
errs = append(errs, fmt.Errorf("remove persistent volume %s: %w", vol.Name, err))
|
|
continue
|
|
}
|
|
removed++
|
|
}
|
|
return removed, errors.Join(errs...)
|
|
}
|
|
|
|
// ConvertMounts converts []Mount to moby mount.Mount for container creation.
|
|
func ConvertMounts(mounts []Mount) []mount.Mount {
|
|
out := make([]mount.Mount, len(mounts))
|
|
for i, m := range mounts {
|
|
out[i] = mount.Mount{
|
|
Type: mount.Type(m.Type),
|
|
Source: m.Source,
|
|
Target: m.Destination,
|
|
}
|
|
}
|
|
return out
|
|
}
|