diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl index e59e999ac..3d4bbed67 100644 --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -12,3 +12,4 @@ {"id":"int-d3c2080053c72c054c32f4d5befd2be0","kind":"field_change","created_at":"2026-07-23T23:08:11.15292258Z","actor":"Sam Catlow","issue_id":"archie-core-5zh.7","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Added internal/taskrun (Request/Response wire types + SubjectForTask) and cmd/archie-agent/taskrun.go: runTask builds a workflow.Registry from the container's own bind-mounted worktree (skillbuild.BuildRegistry, same call the daemon already makes), routes and runs the task's entire workflow via workflow.Route/workflow.Run, using storerpc.Client/forgerpc.Client for Store/Forge and a new hybridTrees (worktreerpc.Client for Prepare/Push, local *worktree.Manager for CommitAll/Diff/ChangedFiles/ChangedLines against the mount). Added config.TaskConfig.ToConfig() to rebuild a usable config.Config from the curated payload. Added storage.WorktreeMountDir constant (/data/worktree) — critical fix: hybridTrees.Prepare discards the RPC response's host-side directory and returns the container's fixed local mount path instead, since archied and archie-agent see the same files at different paths. Wired a queue-grouped core-NATS subscription on archie.taskrun.> in cmd/archie-agent/main.go so this is reachable in the running binary, not just testable in isolation. End-to-end test drives the real 'bootstrap' workflow (no LLM) through real storerpc/forgerpc/worktreerpc servers and a real local git remote, proving the full chain: registry build, route, prepare, commit, push, PR creation, and that the authoritative task state lands in archied's store via RPC. archie-core-5zh.8 (daemon.go handoff) is now unblocked — it owns deciding exactly how/when archied publishes to archie.taskrun.."}} {"id":"int-029f2aa8429c3fd67bdc1541fd78f9c3","kind":"field_change","created_at":"2026-07-23T23:14:48.198047941Z","actor":"Sam Catlow","issue_id":"archie-core-5zh.8","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"daemon.process() now branches on d.ContainerPool != nil: sandboxed tasks get handed to archie-agent via a new runViaAgent method (publishes taskrun.Request to archie.taskrun., awaits taskrun.Response) instead of calling workflow.Route/workflow.Run locally; non-container mode is unchanged. runViaAgent only parks the task itself on transport/setup failure (encode error, no responder, decode error, or an error returned before archie-agent's workflow.Run got to run) — on success, every durable side effect already landed via the storerpc/forgerpc/worktreerpc proxies during archie-agent's run, so runViaAgent just logs. Added agentexec.ProvidersFromConfig (deduped from cmd/archied's former executionProviders). Also fixed a real correctness gap this surfaced: with concurrent multi-task dispatch (u64/g1l) plus containerized archie-agent, the 5zh.7 wildcard queue-group subscription could let one container answer a sibling's taskrun message despite having a different task's worktree mounted. archie-agent now reads its boot-time task.json (container.TaskPayload, already written by daemon.process() before container acquire) and subscribes to its own exact archie.taskrun. subject when present, falling back to the shared queue group only when no task.json exists (dev/pool mode). Full container-mode integration (real Docker) isn't covered by an automated test, consistent with the rest of the container-dependent code — runViaAgent itself is unit-tested directly against embedded NATS, bypassing the Docker-dependent Acquire path. This closes the archie-core-5zh epic: archie-agent now runs an entire task's workflow per invocation instead of one stage per round trip, with Store/Forge/git-push credentials staying daemon-side throughout."}} {"id":"int-49d653540ccc3b48763790e39f72c0b1","kind":"field_change","created_at":"2026-07-23T23:15:04.490104273Z","actor":"Sam Catlow","issue_id":"archie-core-5zh","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"All sub-issues (5zh.1 Store RPC, 5zh.2 Forge RPC, 5zh.3 worktree Prepare/Push RPC, 5zh.5 curated TaskConfig payload, 5zh.6 TaskContext interface refactor, 5zh.7 archie-agent full-task handler, 5zh.9 register RPC servers, 5zh.8 daemon handoff) are closed. Original acceptance criterion met: workflow.Route and workflow.Run execute inside archie-agent, not archied, in sandboxed NATS+container mode; archie-agent handles a full task per invocation. inprocess/subprocess modes unaffected."}} +{"id":"int-a3541f29c68ee77fb14b771c250c0e4a","kind":"field_change","created_at":"2026-07-24T00:20:21.405370187Z","actor":"Sam Catlow","issue_id":"archie-core-j7v","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Persistent container/worktree storage now has configured TTL expiry. Repos opting into persistent_storage retain an Archie-owned Docker repo volume and reuse a per-repo bare Git object cache to avoid repeated full network clones while keeping issue worktrees isolated via dissociated clones. containers.volume_ttl defaults to 72h; daemon startup/cycles automatically remove expired owned repo volumes and Git caches. Tests cover config default/validation, Docker ownership labels and selective expiry, cache reuse/isolation/expiry/path safety, and daemon TTL wiring. Full race suite, vet, build, formatting, and scoped staticcheck pass."}} diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 64687edc9..759af598a 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -12,5 +12,6 @@ {"_type":"issue","id":"archie-core-5zh","title":"Promote archie-agent to per-task process; move workflow routing out of archied","description":"cmd/archie-agent exists but only executes one workflow stage per invocation over the internal/agentexec stdin/stdout JSON protocol (v1). Phase 1 of the PRD requires promoting it to the PRD's per-task process model, moving workflow.Route/workflow.Run out of archied and into archie-agent.","acceptance_criteria":"workflow.Route and workflow.Run execute inside archie-agent, not archied; archie-agent handles a full task (not a single stage) per invocation.","status":"closed","priority":2,"issue_type":"task","assignee":"Sam Catlow","owner":"sam@catlow.tech","created_at":"2026-07-23T19:36:32Z","created_by":"Sam Catlow","updated_at":"2026-07-23T23:15:04Z","started_at":"2026-07-23T21:51:46Z","closed_at":"2026-07-23T23:15:04Z","close_reason":"All sub-issues (5zh.1 Store RPC, 5zh.2 Forge RPC, 5zh.3 worktree Prepare/Push RPC, 5zh.5 curated TaskConfig payload, 5zh.6 TaskContext interface refactor, 5zh.7 archie-agent full-task handler, 5zh.9 register RPC servers, 5zh.8 daemon handoff) are closed. Original acceptance criterion met: workflow.Route and workflow.Run execute inside archie-agent, not archied, in sandboxed NATS+container mode; archie-agent handles a full task per invocation. inprocess/subprocess modes unaffected.","dependencies":[{"issue_id":"archie-core-5zh","depends_on_id":"archie-core-l1q","type":"blocks","created_at":"2026-07-24T05:36:36Z","created_by":"Sam Catlow","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"archie-core-l1q","title":"NATS transport: JetStream task queue + request-reply between daemon and archie-agent","description":"Phase 1 of docs/prds/archie-core-v2-architecture.md. internal/events.Bus (in-process pub/sub, already feeding SQLite + SSE) is the natural seam to replace the transport under. Daemon publishes tasks, archie-agent consumes, results return via NATS. Preserve current inprocess mode via [agent] mode default for local dev.","design":"Add optional [nats] config block (url, token_env), default anonymous auth for single-node, credentials required for multi-node per PRD sec.9 open question 2.","acceptance_criteria":"Daemon can dispatch a task to archie-agent over NATS JetStream and receive results via request-reply; inprocess mode still works unchanged.","status":"closed","priority":2,"issue_type":"feature","assignee":"Sam Catlow","owner":"sam@catlow.tech","created_at":"2026-07-23T19:36:31Z","created_by":"Sam Catlow","updated_at":"2026-07-23T21:49:40Z","started_at":"2026-07-23T21:43:05Z","closed_at":"2026-07-23T21:49:40Z","close_reason":"Completed JetStream task dispatch and NATS agent request/reply, including token_env authentication for archied and canonical NATS_URL/NATS_TOKEN propagation to container workers; anonymous local mode remains supported and race-tested.","dependencies":[{"issue_id":"archie-core-l1q","depends_on_id":"archie-core-u64","type":"blocks","created_at":"2026-07-24T05:36:36Z","created_by":"Sam Catlow","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"archie-core-rig","title":"Audit archie-core repo for bugs/improvements, file as Gitea issues","description":"Fan out exploration subagents across internal/ subsystems to find bugs, tech debt, and enhancement opportunities. Each confirmed finding gets filed as a Gitea issue on git.catlow.cloud/sam/archie-core.","status":"closed","priority":2,"issue_type":"task","assignee":"Sam Catlow","owner":"sam@catlow.tech","created_at":"2026-07-22T16:24:07Z","created_by":"Sam Catlow","updated_at":"2026-07-22T16:37:16Z","started_at":"2026-07-22T16:24:09Z","closed_at":"2026-07-22T16:37:16Z","close_reason":"Audit complete: 58 Gitea issues (#3-#60) filed at git.catlow.cloud/sam/archie-core. 60 raw findings, 59 verified, 58 successfully filed. 8 agents hit session limit at end (webui/workflow/worktree file-phase).","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"archie-core-756","title":"Restore a clean repository-wide golangci-lint baseline","description":"Repository-wide golangci-lint currently reports 39 pre-existing findings (32 errcheck and 7 staticcheck) across NATS/RPC/container code and older tests. Remediate the findings without weakening lint configuration so golangci-lint run ./... is a reliable required gate.","acceptance_criteria":"golangci-lint run ./... exits successfully with no findings; affected cleanup/error paths handle or explicitly document ignored errors; existing race tests remain green.","status":"open","priority":3,"issue_type":"task","owner":"sam@catlow.tech","created_at":"2026-07-24T00:20:00Z","created_by":"Sam Catlow","updated_at":"2026-07-24T00:20:00Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"archie-core-g1l","title":"Per-repo allow_concurrent override for same-repo multi-agent work","description":"Concurrent agents on the same repo are blocked by default (worktree conflict). PRD sec.9 open question 5 proposes a per-repo config override (allow_concurrent = true) for repos where parallel work is safe (different base branches, different packages). Deferred to Phase 2 in the PRD; depends on the daemon supporting concurrent task dispatch at all.","acceptance_criteria":"Repos with allow_concurrent = true in [[repos]] config can have multiple agents working simultaneously without worktree collisions; repos without it keep current serialized behavior.","status":"closed","priority":3,"issue_type":"feature","assignee":"Sam Catlow","owner":"sam@catlow.tech","created_at":"2026-07-23T19:36:32Z","created_by":"Sam Catlow","updated_at":"2026-07-23T21:46:53Z","started_at":"2026-07-23T21:43:14Z","closed_at":"2026-07-23T21:46:53Z","close_reason":"Added config.Repo.AllowConcurrent (allow_concurrent toml key) and threaded it through taskDispatcher so opted-in repos skip the default FIFO same-repo serialization while still respecting containers.max_concurrency. Worktrees are already keyed per issue number, so no on-disk collision. Race-tested dispatcher coverage added.","dependencies":[{"issue_id":"archie-core-g1l","depends_on_id":"archie-core-u64","type":"blocks","created_at":"2026-07-24T05:36:37Z","created_by":"Sam Catlow","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"archie-core-j7v","title":"Persistent volumes with TTL for container/worktree storage","description":"Phase 2 of docs/prds/archie-core-v2-architecture.md. Current worktree lifecycle is fresh-clone-per-task with status-driven cleanup, no TTL. Add persistent volume support with TTL-based expiry to avoid repeated full clones for repeated work on the same repo.","acceptance_criteria":"Container/worktree storage can persist across tasks for the same repo up to a configured TTL, with automatic cleanup after expiry.","status":"open","priority":3,"issue_type":"feature","owner":"sam@catlow.tech","created_at":"2026-07-23T19:36:32Z","created_by":"Sam Catlow","updated_at":"2026-07-23T19:36:32Z","dependencies":[{"issue_id":"archie-core-j7v","depends_on_id":"archie-core-358","type":"blocks","created_at":"2026-07-24T05:36:37Z","created_by":"Sam Catlow","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"archie-core-j7v","title":"Persistent volumes with TTL for container/worktree storage","description":"Phase 2 of docs/prds/archie-core-v2-architecture.md. Current worktree lifecycle is fresh-clone-per-task with status-driven cleanup, no TTL. Add persistent volume support with TTL-based expiry to avoid repeated full clones for repeated work on the same repo.","acceptance_criteria":"Container/worktree storage can persist across tasks for the same repo up to a configured TTL, with automatic cleanup after expiry.","status":"in_progress","priority":3,"issue_type":"feature","assignee":"Sam Catlow","owner":"sam@catlow.tech","created_at":"2026-07-23T19:36:32Z","created_by":"Sam Catlow","updated_at":"2026-07-23T23:33:54Z","started_at":"2026-07-23T23:33:54Z","dependencies":[{"issue_id":"archie-core-j7v","depends_on_id":"archie-core-358","type":"blocks","created_at":"2026-07-24T05:36:37Z","created_by":"Sam Catlow","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} diff --git a/cmd/archie-agent/taskrun_test.go b/cmd/archie-agent/taskrun_test.go index e8437f82f..59cc24df2 100644 --- a/cmd/archie-agent/taskrun_test.go +++ b/cmd/archie-agent/taskrun_test.go @@ -68,21 +68,27 @@ func (f *prCapturingForge) AcceptInvitations(context.Context) error func (f *prCapturingForge) AssignedIssues(context.Context, string, string, string) ([]forge.Issue, error) { panic("unexpected call") } + func (f *prCapturingForge) IssuesWithLabel(context.Context, string, string, string) ([]forge.Issue, error) { panic("unexpected call") } + func (f *prCapturingForge) RepliesAfter(context.Context, string, string, int, int64, string) ([]forge.Reply, error) { panic("unexpected call") } + func (f *prCapturingForge) PRState(context.Context, string, string, int) (string, error) { panic("unexpected call") } + func (f *prCapturingForge) CreateIssue(context.Context, string, string, string, string, []string) (int, error) { panic("unexpected call") } + func (f *prCapturingForge) React(context.Context, string, string, int, string) error { panic("unexpected call") } + func (f *prCapturingForge) VerifyPush(context.Context, string, string) error { panic("unexpected call") } diff --git a/cmd/archied/main_test.go b/cmd/archied/main_test.go index 7cab72043..e6fbbcc98 100644 --- a/cmd/archied/main_test.go +++ b/cmd/archied/main_test.go @@ -35,15 +35,19 @@ func (stubForge) AcceptInvitations(context.Context) error func (stubForge) AssignedIssues(context.Context, string, string, string) ([]forge.Issue, error) { panic("unexpected call") } + func (stubForge) IssuesWithLabel(context.Context, string, string, string) ([]forge.Issue, error) { panic("unexpected call") } + func (stubForge) RepliesAfter(context.Context, string, string, int, int64, string) ([]forge.Reply, error) { panic("unexpected call") } + func (stubForge) PRState(context.Context, string, string, int) (string, error) { panic("unexpected call") } + func (stubForge) CreateIssue(context.Context, string, string, string, string, []string) (int, error) { panic("unexpected call") } diff --git a/config.example.toml b/config.example.toml index 2d0c8d55a..7d50e1b11 100644 --- a/config.example.toml +++ b/config.example.toml @@ -69,6 +69,7 @@ gate_max_failures = 5 # image = "archie-agent:latest" # Docker image to run # max_concurrency = 4 # max simultaneous tasks/containers # max_uptime = "60m" # container lifetime cap +# volume_ttl = "72h" # max age for persistent repo volumes and Git clone caches # pull_policy = "missing" # "missing" or "always" [[repos]] @@ -91,3 +92,4 @@ gate = [ # FIFO-serializing them; only safe when concurrent worktrees won't collide # (e.g. different base branches or packages per task). Still bounded by # [containers].max_concurrency globally. +# persistent_storage = true # reuse a per-repo Docker volume and Git object cache until volume_ttl diff --git a/docs/prds/archie-core-v2-architecture.md b/docs/prds/archie-core-v2-architecture.md index eaceab791..addeac3ce 100644 --- a/docs/prds/archie-core-v2-architecture.md +++ b/docs/prds/archie-core-v2-architecture.md @@ -16,14 +16,14 @@ This document describes a target end state. The table below is the ground truth; | Sandbox / "daemon never runs untrusted code" | **Partial** | Default mode is still `inprocess` for development. Docker sandbox mode (`agent.mode = "nats"` + `containers.enabled = true`) runs each task in an ephemeral Docker container with workspace bind-mounted at `/workspace`. `internal/container/pool.go` manages full lifecycle: pull, create, start, release, orphan recovery. Daemon dispatch and container lifecycle both enforce `max_concurrency`; same-repo tasks remain serialized. | | NATS / JetStream messaging | **Implemented** | `internal/nats/client.go`: JetStream connect, stream creation (WorkQueue retention, file storage), pull consumer (durable, ack-wait 5min, max-deliver 3, dedup by Msg-Id), `PublishTask`, `Fetch`, `NewReplyInbox`, and optional token authentication. Subject hierarchy lives in `internal/nats/subjects.go`; `internal/agentexec/nats.go` implements request/reply agent execution. | | Docker container sandbox | **Implemented** | `internal/container/pool.go` (210 lines): Docker API client, image pull (always/missing/never), container create with workspace bind mount, start, release (stop+remove), orphan recovery on daemon restart. `Dockerfile` at repo root builds archie-agent fat image. `[containers]` config block: enabled, image, max_concurrency, max_uptime, pull_policy. `agent.mode = "nats"` required when containers enabled. | -| `/data/` volume, `task.json` | **Partial** | `container.WriteTaskJSON()` writes task payload to the worktree before container acquire. The container pool bind-mounts the worktree at `/data/worktree`. Task payload travels over NATS as the primary channel; `/data/task.json` is a boot-time brief for the container. Worktrees are still ephemeral fresh clones per task. | +| `/data/` volume, `task.json` | **Partial** | `container.WriteTaskJSON()` writes task payload to the worktree before container acquire. The container pool bind-mounts the isolated task worktree at `/data/worktree`; configured repos also receive a persistent `/data/repo` Docker volume. Task payload travels over NATS as the primary channel; `/data/task.json` is a boot-time brief for the container. | | `[containers]` config block | **Implemented** | `ContainerConfig` struct in config.go: enabled, image, max_concurrency, max_uptime, pull_policy. Validated on load. | | Yaegi extensibility (gates, custom stages, skill scripts) | **Implemented** | Three surfaces: `.archie/gate.go`, `.archie/stages/*.go`, skill scripts via `run_go_script` tool. All in-process via Yaegi with generated symbol tables and panic recovery. Full detail in `docs/prds/yaegi-plugin-system.md`. | | Layer 1: skill-bundled plugins | **Implemented** | `skill.DiscoverPlugins()` scans `.agents/skills//plugins/*.go`. `skill.Plugin.Run()` executes them via Yaegi. `skillbuild.BuildWorkflow()` constructs `workflow.Workflow` from plugin-defined stages. `TaskContext.SkillPlugins` makes them available during stage execution. `skillbuild.BuildRegistry()` replaces the hardcoded `workflow.Registry` with catalog-driven construction — skills define workflows, plugins define stages. | | Layer 2: core daemon plugins | **Partial** | `internal/plugin/plugin.go`: `Plugin` interface (`Name()`, `Version()`), `Registry` type, `LoadDir()` using Yaegi to interpret `.go` files. No `~/.config/archie/plugins/` loader wired into `cmd/archied/main.go` yet — the interface and registry exist but are not connected to the daemon's startup. | | Skills (agentskills.io spec) | **Implemented** | `internal/skill/skill.go`: parses SKILL.md YAML frontmatter, discovers skills from `.agents/skills/*/SKILL.md`, extracts body. Skills follow `archie-wf-*` naming convention. Progressive disclosure: `Catalog()` (Tier 1 — frontmatter-only scan, ~100 tokens per entry), `LoadBody()` (Tier 2 — full body on activation), `Discover()` (Tier 3 — full parse with plugins). `metadata.archie.workflow` declares skill→workflow affinity. `skill.SkillForWorkflow()` is catalog-driven, no hardcoded maps. `skillbuild.BuildRegistry()` constructs the daemon's workflow registry from the catalog. | -| Worktree/repo management | **Implemented, ephemeral** | Fresh `git clone` per task attempt, no shared cache, no persistence, no TTL. | -| Config schema | **Closer to sec.4** | `[agent]` block: mode (inprocess/subprocess/nats) + command + env. `[nats]` block: url + token_env. `[containers]` block: enabled, image, max_concurrency, max_uptime, pull_policy. No per-repo persistent_storage override yet. | +| Worktree/repo management | **Implemented** | Each issue keeps an isolated worktree. Repos with `persistent_storage = true` use a per-repo bare Git object cache to avoid repeated full network clones; caches and Archie-owned repo volumes expire via `containers.volume_ttl`. | +| Config schema | **Closer to sec.4** | `[agent]` block: mode (inprocess/subprocess/nats) + command + env. `[nats]` block: url + token_env. `[containers]` block: enabled, image, max_concurrency, max_uptime, volume_ttl, pull_policy. Per-repo `persistent_storage` is supported as a boolean Docker-volume/Git-cache opt-in. | **Net read post-1dca419:** Phases 1, 2, and 3 are substantially implemented. NATS, Docker sandbox, skill parsing, skill-bundled plugins (Layer 1), skill progressive disclosure, and catalog-driven workflow construction (`skillbuild.BuildRegistry()`) all landed. The remaining aspirational work is Layer 2 daemon plugin loader (interface exists, not wired), pluggable storage backends, and per-worktree dynamic registry augmentation. @@ -117,15 +117,10 @@ retry logic -- NATS handles it. ## 3. Persistent volume -> **Status: aspirational, zero code.** No `/data/` directory convention, no `task.json` file, no -> `session.jsonl`/`memory.jsonl` exist on disk anywhere. The closest thing today is `store.Task` (a SQLite row, passed -> by function call, different shape from the JSON below -- e.g. `Labels` is a comma-separated string, not an array, and -> there's no nested `config`/`nats` object) plus the `agentexec` wire protocol, which passes an `Invocation` directly -> over the subprocess's stdin as JSON rather than through a mounted volume. Worktrees are real but ephemeral: -> `internal/worktree.Manager.Prepare` does a fresh `git clone` per task attempt into `WorkDir/-/issue-`, -> removing any prior directory first -- no shared clone cache, no persistence across attempts, no TTL-based cleanup -> (cleanup today is purely lifecycle-status-driven: removed on terminal merged/rejected outcomes, kept indefinitely -> while parked). +> **Status: partially implemented.** `task.json`, `/data/worktree`, optional per-repo `/data/repo` Docker volumes, +> ecosystem cache volumes, and TTL cleanup exist. Task worktrees remain isolated by issue, while configured repos use a +> per-repo bare Git object cache to seed clones without repeating full network transfers. `session.jsonl`, +> `memory.jsonl`, and staged plugin volumes remain unimplemented. ### /data/ layout @@ -185,14 +180,10 @@ The daemon writes `task.json` to the volume before the container starts. The age ## 4. Container management -> **Status: aspirational, zero code.** `internal/config/config.go` has no `[containers]`, `[containers.image]`, or -> `[containers.persistent_storage]` sections, and `Repo` has no per-repo `persistent_storage` override. The real config -> today is a much smaller `[agent]` block: `mode` (`"inprocess"` default, or `"subprocess"`), `command` (defaults to the -> PATH-resolved `archie-agent` binary name, not a container image), and `env`. There is no Docker/container library -> dependency in `go.mod`, no `Dockerfile.agent`, and no spawn/kill/grace-period lifecycle code -- `SubprocessRunner` -> uses plain `os/exec.CommandContext` with an env-allowlist, running as the daemon's own OS user. `config.example.toml` -> already documents this gap directly: _"The worker still runs as the daemon user until the Docker sandbox phase -> lands."_ +> **Status: implemented with a flatter schema than the sketch below.** `[containers]` configures the image, +> concurrency, uptime, pull policy, and persistent-storage TTL. `Repo.PersistentStorage` opts a repository into the +> per-repo Docker volume and Git object cache. The backend abstraction exists in `internal/storage`; additional storage +> types and structured per-backend configuration remain future work. ### Daemon config @@ -459,8 +450,8 @@ The agent writes whatever it wants to `response`. The daemon reviews it. No agen - Docker agent image (fat, with tools) -- not started - Daemon container lifecycle: spawn, volume create, env inject, kill -- not started -- Persistent volumes with TTL -- not started (current worktree lifecycle is fresh-clone-per-task with status-driven - cleanup, no TTL) +- ~~Persistent volumes with TTL~~ -- **done**: configured repos receive a persistent Docker volume plus a per-repo Git + object cache; both are automatically reaped using `containers.volume_ttl`. - ~~`max_concurrency` enforcement~~ -- **done**: SQLite and NATS drains dispatch different repositories concurrently up to `[containers].max_concurrency`; same-repo tasks remain serialized by default. - Grace period `max_uptime` -- not started diff --git a/internal/config/config.go b/internal/config/config.go index e59153b02..104b69e0a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -363,6 +363,10 @@ type ContainerConfig struct { MaxConcurrency int `toml:"max_concurrency"` // MaxUptime caps a container's lifetime before recycling. MaxUptime Duration `toml:"max_uptime"` + // VolumeTTL is the maximum age of persistent per-repo storage before + // automatic cleanup. It also bounds the host-side Git object cache used + // to seed isolated task worktrees. + VolumeTTL Duration `toml:"volume_ttl"` // PullPolicy controls image pulling: "missing" (default) or "always". PullPolicy string `toml:"pull_policy"` } @@ -511,6 +515,12 @@ func Load(path string) (Config, error) { if cfg.Containers.MaxUptime == 0 { cfg.Containers.MaxUptime = Duration(60 * time.Minute) } + if cfg.Containers.VolumeTTL < 0 { + return cfg, errors.New("config: containers.volume_ttl must not be negative") + } + if cfg.Containers.VolumeTTL == 0 { + cfg.Containers.VolumeTTL = Duration(72 * time.Hour) + } if cfg.Containers.PullPolicy == "" { cfg.Containers.PullPolicy = "missing" } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 1e8f22b9b..8c5e1f112 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -421,6 +421,54 @@ func TestLoadAgentDefaultsAndOverrides(t *testing.T) { } } +func TestLoadContainerVolumeTTL(t *testing.T) { + tests := []struct { + name string + ttl string + wantTTL time.Duration + }{ + {name: "default", wantTTL: 72 * time.Hour}, + {name: "explicit", ttl: "volume_ttl = \"6h\"\n", wantTTL: 6 * time.Hour}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + contents := "bot_user = \"archie\"\n" + + "[agent]\nmode = \"nats\"\n" + + "[nats]\nurl = \"nats://localhost:4222\"\n" + + "[containers]\nenabled = true\nimage = \"archie-agent:test\"\n" + tt.ttl + + "[[repos]]\nowner = \"acme\"\nname = \"app\"\npersistent_storage = true\n" + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if got := cfg.Containers.VolumeTTL.Std(); got != tt.wantTTL { + t.Fatalf("Containers.VolumeTTL = %s, want %s", got, tt.wantTTL) + } + }) + } +} + +func TestLoadRejectsNegativeContainerVolumeTTL(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + contents := "bot_user = \"archie\"\n" + + "[agent]\nmode = \"nats\"\n" + + "[nats]\nurl = \"nats://localhost:4222\"\n" + + "[containers]\nenabled = true\nimage = \"archie-agent:test\"\nvolume_ttl = \"-1h\"\n" + + "[[repos]]\nowner = \"acme\"\nname = \"app\"\npersistent_storage = true\n" + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + + if _, err := Load(path); err == nil { + t.Fatal("Load() accepted a negative containers.volume_ttl") + } +} + func TestExampleConfigLoads(t *testing.T) { cfg, err := Load(filepath.Join("..", "..", "config.example.toml")) if err != nil { diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 0d23c32cd..cf8a510d0 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -76,6 +76,7 @@ func (d *Daemon) emit(e events.Event) { // Startup runs crash recovery and access verification once. func (d *Daemon) Startup(ctx context.Context) error { + d.cleanupExpiredStorage(ctx) n, err := d.Store.RecoverStale(ctx) if err != nil { return err @@ -112,6 +113,7 @@ func (d *Daemon) Run(ctx context.Context) error { // reconcile open PRs, act on human replies to waiting tasks, then // process queued tasks concurrently up to containers.max_concurrency. func (d *Daemon) Cycle(ctx context.Context) { + d.cleanupExpiredStorage(ctx) d.poll(ctx) d.reconcilePRs(ctx) d.checkWaiting(ctx) @@ -122,6 +124,31 @@ func (d *Daemon) Cycle(ctx context.Context) { } } +func (d *Daemon) cleanupExpiredStorage(ctx context.Context) { + ttl := d.Cfg.Containers.VolumeTTL.Std() + if ttl <= 0 { + return + } + if d.Storage != nil { + n, err := d.Storage.CleanupExpired(ctx, ttl) + if err != nil { + d.Log.Warn("persistent volume cleanup failed", "err", err) + } + if n > 0 { + d.Log.Info("expired persistent volumes removed", "count", n) + } + } + if d.Trees != nil { + n, err := d.Trees.CleanupExpiredCaches(ttl) + if err != nil { + d.Log.Warn("repo cache cleanup failed", "err", err) + } + if n > 0 { + d.Log.Info("expired repo caches removed", "count", n) + } + } +} + // drainSQLite claims queued tasks from SQLite and dispatches them concurrently. func (d *Daemon) drainSQLite(ctx context.Context) { dispatcher := newTaskDispatcher(d.Cfg.Containers.MaxConcurrency, d.allowConcurrentFor) @@ -634,7 +661,16 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) { // Clone the worktree before Docker mount setup. The container // binds /data/worktree to workDir on the host — the directory // must exist before Acquire is called. - _, branch, err := d.Trees.Prepare(ctx, task.Owner, task.Repo, repo.Base, task.IssueNumber, task.Title, task.Body, task.Labels) + var branch string + var err error + if repo.PersistentStorage && d.Cfg.Containers.VolumeTTL.Std() > 0 { + _, branch, err = d.Trees.PreparePersistent( + ctx, task.Owner, task.Repo, repo.Base, task.IssueNumber, + task.Title, task.Body, task.Labels, d.Cfg.Containers.VolumeTTL.Std(), + ) + } else { + _, branch, err = d.Trees.Prepare(ctx, task.Owner, task.Repo, repo.Base, task.IssueNumber, task.Title, task.Body, task.Labels) + } if err != nil { d.Log.Error("worktree prepare failed", "err", err) _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "worktree prepare failed: "+err.Error()) diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index faaa4fcfa..0eba1682a 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -18,10 +18,45 @@ import ( "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/forge" arnats "github.com/samcharles93/archie-core/internal/nats" + "github.com/samcharles93/archie-core/internal/storage" "github.com/samcharles93/archie-core/internal/store" "github.com/samcharles93/archie-core/internal/taskrun" ) +type cleanupStorage struct { + ttl time.Duration +} + +func (s *cleanupStorage) Setup(context.Context, storage.TaskRef) ([]storage.Mount, error) { + return nil, nil +} + +func (s *cleanupStorage) Teardown(context.Context, storage.TaskRef) error { + return nil +} + +func (s *cleanupStorage) CleanupExpired(_ context.Context, ttl time.Duration) (int, error) { + s.ttl = ttl + return 2, nil +} + +func TestCleanupExpiredStorageUsesConfiguredTTL(t *testing.T) { + backend := &cleanupStorage{} + d := &Daemon{ + Cfg: config.Config{ + Containers: config.ContainerConfig{VolumeTTL: config.Duration(6 * time.Hour)}, + }, + Storage: backend, + Log: slog.New(slog.DiscardHandler), + } + + d.cleanupExpiredStorage(context.Background()) + + if backend.ttl != 6*time.Hour { + t.Fatalf("CleanupExpired ttl = %s, want 6h", backend.ttl) + } +} + func TestTaskDispatcherEnforcesMaxConcurrency(t *testing.T) { t.Parallel() diff --git a/internal/forgerpc/forgerpc_test.go b/internal/forgerpc/forgerpc_test.go index 79e6b90f1..a2a1cd138 100644 --- a/internal/forgerpc/forgerpc_test.go +++ b/internal/forgerpc/forgerpc_test.go @@ -94,18 +94,23 @@ func (f *fakeForge) AcceptInvitations(context.Context) error { panic("unexpected func (f *fakeForge) AssignedIssues(context.Context, string, string, string) ([]forge.Issue, error) { panic("unexpected call") } + func (f *fakeForge) IssuesWithLabel(context.Context, string, string, string) ([]forge.Issue, error) { panic("unexpected call") } + func (f *fakeForge) RepliesAfter(context.Context, string, string, int, int64, string) ([]forge.Reply, error) { panic("unexpected call") } + func (f *fakeForge) PRState(context.Context, string, string, int) (string, error) { panic("unexpected call") } + func (f *fakeForge) CreateIssue(context.Context, string, string, string, string, []string) (int, error) { panic("unexpected call") } + func (f *fakeForge) React(context.Context, string, string, int, string) error { panic("unexpected call") } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 704e723d4..1cc37812d 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -30,9 +30,11 @@ package storage import ( "context" + "errors" "fmt" "sort" "strings" + "time" "github.com/moby/moby/api/types/mount" "github.com/moby/moby/client" @@ -76,6 +78,9 @@ type Backend interface { Setup(ctx context.Context, task TaskRef) ([]Mount, error) // Teardown cleans up ephemeral storage. Cache volumes survive. Teardown(ctx context.Context, task TaskRef) error + // CleanupExpired removes persistent per-repo storage older than ttl. + // Shared ecosystem caches and storage not owned by Archie survive. + CleanupExpired(ctx context.Context, ttl time.Duration) (int, error) } // ── cache mounts per ecosystem ───────────────────────────────────────── @@ -155,8 +160,15 @@ func (e *ErrVolumeCreate) Unwrap() error { return e.Cause } // DockerBackend implements Backend for Docker via the moby client. type DockerBackend struct { cli *client.Client + now func() time.Time } +const ( + storageLabel = "com.samcharles93.archie.storage" + storageKindRepo = "repo" + storageKindCache = "cache" +) + // NewDockerBackend creates a Docker storage backend. func NewDockerBackend(cli *client.Client) *DockerBackend { return &DockerBackend{cli: cli} @@ -170,7 +182,7 @@ func NewDockerBackend(cli *client.Client) *DockerBackend { // creating a volume. The returned mounts will fail at container create // time if Docker cannot find the volume. Production code always sets // cli via NewDockerBackend. -func (d *DockerBackend) ensureVolume(ctx context.Context, name string) error { +func (d *DockerBackend) ensureVolume(ctx context.Context, name string, labels map[string]string) error { if d.cli == nil { return nil // nil client for testing — volumes must be pre-created } @@ -183,6 +195,7 @@ func (d *DockerBackend) ensureVolume(ctx context.Context, name string) error { _, err = d.cli.VolumeCreate(ctx, client.VolumeCreateOptions{ Name: name, Driver: "local", + Labels: labels, }) if err != nil { // TOCTOU: a concurrent caller may have created the volume between @@ -217,7 +230,7 @@ func (d *DockerBackend) Setup(ctx context.Context, task TaskRef) ([]Mount, error // Per-repo persistent volume. if task.PersistentStorage && task.Owner != "" && task.Repo != "" { volName := repoVolumeName(task.Owner, task.Repo) - if err := d.ensureVolume(ctx, volName); err != nil { + if err := d.ensureVolume(ctx, volName, map[string]string{storageLabel: storageKindRepo}); err != nil { return nil, err } mounts = append(mounts, Mount{ @@ -229,7 +242,7 @@ func (d *DockerBackend) Setup(ctx context.Context, task TaskRef) ([]Mount, error caches := cacheMounts(task.Ecosystem) for _, m := range caches { - if err := d.ensureVolume(ctx, m.Source); err != nil { + if err := d.ensureVolume(ctx, m.Source, map[string]string{storageLabel: storageKindCache}); err != nil { return nil, err } } @@ -244,6 +257,51 @@ func (d *DockerBackend) Teardown(ctx context.Context, task TaskRef) error { return nil } +// CleanupExpired removes Archie-owned per-repo volumes whose creation time is +// at least ttl old. Docker volume labels are immutable, so the TTL is a hard +// maximum age rather than an idle lease. Removal is non-forced: a volume still +// attached to a running container is retained and retried on the next sweep. +func (d *DockerBackend) CleanupExpired(ctx context.Context, ttl time.Duration) (int, error) { + if d.cli == nil || ttl <= 0 { + return 0, nil + } + result, err := d.cli.VolumeList(ctx, client.VolumeListOptions{ + Filters: client.Filters{}.Add("label", storageLabel+"="+storageKindRepo), + }) + if err != nil { + return 0, fmt.Errorf("list persistent volumes: %w", err) + } + + now := time.Now() + if d.now != nil { + now = d.now() + } + var ( + removed int + errs []error + ) + for _, vol := range result.Items { + // Defend against Docker implementations that ignore label filters. + if vol.Labels[storageLabel] != storageKindRepo { + continue + } + created, err := time.Parse(time.RFC3339Nano, vol.CreatedAt) + if err != nil { + errs = append(errs, fmt.Errorf("persistent volume %s created_at %q: %w", vol.Name, vol.CreatedAt, err)) + continue + } + if now.Sub(created) < ttl { + continue + } + if _, err := d.cli.VolumeRemove(ctx, vol.Name, client.VolumeRemoveOptions{}); err != nil { + errs = append(errs, fmt.Errorf("remove persistent volume %s: %w", vol.Name, err)) + continue + } + removed++ + } + return removed, errors.Join(errs...) +} + // ConvertMounts converts []Mount to moby mount.Mount for container creation. func ConvertMounts(mounts []Mount) []mount.Mount { out := make([]mount.Mount, len(mounts)) diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 8e660ffa1..e30316bce 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -2,10 +2,19 @@ package storage import ( "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" "os" "slices" "sort" + "strings" "testing" + "time" + + "github.com/moby/moby/api/types/volume" + "github.com/moby/moby/client" ) // ── interface contract ─────────────────────────────────────────────── @@ -370,6 +379,98 @@ func TestRepoVolumeName(t *testing.T) { } } +func TestDockerBackendLabelsPersistentRepoVolumes(t *testing.T) { + var createBody string + cli := dockerTestClient(t, func(req *http.Request) (*http.Response, error) { + path := dockerAPIPath(req.URL.Path) + switch { + case req.Method == http.MethodGet: + return dockerResponse(http.StatusNotFound, `{"message":"not found"}`), nil + case req.Method == http.MethodPost && path == "/volumes/create": + data, readErr := io.ReadAll(req.Body) + if readErr != nil { + return nil, readErr + } + createBody = string(data) + return dockerResponse(http.StatusCreated, `{"Name":"archie-repo-alice-demo","Driver":"local","Labels":{}}`), nil + default: + return dockerResponse(http.StatusNotFound, `{"message":"unexpected request"}`), nil + } + }) + + backend := NewDockerBackend(cli) + if _, err := backend.Setup(context.Background(), TaskRef{ + WorktreeDir: t.TempDir(), + Ecosystem: "custom", + PersistentStorage: true, + Owner: "alice", + Repo: "demo", + }); err != nil { + t.Fatal(err) + } + if !strings.Contains(createBody, `"com.samcharles93.archie.storage":"repo"`) { + t.Fatalf("persistent volume create body lacks Archie repo label: %s", createBody) + } +} + +func TestDockerBackendCleanupExpiredOnlyRemovesOwnedRepoVolumes(t *testing.T) { + now := time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC) + listBody, err := json.Marshal(volume.ListResponse{Volumes: []volume.Volume{ + { + Name: "archie-repo-old", + CreatedAt: now.Add(-25 * time.Hour).Format(time.RFC3339Nano), + Labels: map[string]string{"com.samcharles93.archie.storage": "repo"}, + }, + { + Name: "archie-repo-fresh", + CreatedAt: now.Add(-23 * time.Hour).Format(time.RFC3339Nano), + Labels: map[string]string{"com.samcharles93.archie.storage": "repo"}, + }, + { + Name: "archie-cache-old", + CreatedAt: now.Add(-100 * time.Hour).Format(time.RFC3339Nano), + Labels: map[string]string{"com.samcharles93.archie.storage": "cache"}, + }, + { + Name: "operator-volume", + CreatedAt: now.Add(-100 * time.Hour).Format(time.RFC3339Nano), + }, + }}) + if err != nil { + t.Fatal(err) + } + + var removed []string + cli := dockerTestClient(t, func(req *http.Request) (*http.Response, error) { + path := dockerAPIPath(req.URL.Path) + switch req.Method { + case http.MethodGet: + if !strings.Contains(req.URL.Query().Get("filters"), "com.samcharles93.archie.storage") { + t.Errorf("volume list did not filter by Archie ownership: %s", req.URL.RawQuery) + } + return dockerResponse(http.StatusOK, string(listBody)), nil + case http.MethodDelete: + removed = append(removed, strings.TrimPrefix(path, "/volumes/")) + return dockerResponse(http.StatusNoContent, ""), nil + default: + return dockerResponse(http.StatusNotFound, `{"message":"unexpected request"}`), nil + } + }) + + backend := NewDockerBackend(cli) + backend.now = func() time.Time { return now } + count, err := backend.CleanupExpired(context.Background(), 24*time.Hour) + if err != nil { + t.Fatal(err) + } + if count != 1 { + t.Fatalf("CleanupExpired count = %d, want 1", count) + } + if !slices.Equal(removed, []string{"archie-repo-old"}) { + t.Fatalf("removed volumes = %v, want only expired owned repo volume", removed) + } +} + func TestCacheMountsForTypeScript(t *testing.T) { // S4: typescript ecosystem must share node + pnpm cache volumes. mounts := cacheMounts("typescript") @@ -419,7 +520,7 @@ func TestEnsureVolumeAlreadyExistsIsNotError(t *testing.T) { // something OTHER than "already exists". For now, verify that // ensureVolume with nil client returns nil (existing behavior). backend := &DockerBackend{} - err := backend.ensureVolume(context.Background(), "test-volume") + err := backend.ensureVolume(context.Background(), "test-volume", nil) if err != nil { t.Error("ensureVolume with nil client should not error:", err) } @@ -438,3 +539,51 @@ func mountDestinations(mounts []Mount) []string { func contains(slice []string, s string) bool { return slices.Contains(slice, s) } + +func dockerResponse(status int, body string) *http.Response { + return &http.Response{ + StatusCode: status, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(body)), + } +} + +func dockerTestClient(t *testing.T, roundTrip func(*http.Request) (*http.Response, error)) *client.Client { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + resp, err := roundTrip(req) + if err != nil { + t.Errorf("Docker test handler: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + for key, values := range resp.Header { + for _, value := range values { + w.Header().Add(key, value) + } + } + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, resp.Body); err != nil { + t.Errorf("Docker test response write: %v", err) + } + })) + t.Cleanup(srv.Close) + + cli, err := client.New( + client.WithHost(srv.URL), + client.WithHTTPClient(srv.Client()), + client.WithAPIVersion("1.55"), + ) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cli.Close() }) + return cli +} + +func dockerAPIPath(path string) string { + if strings.HasPrefix(path, "/v1.55/") { + return strings.TrimPrefix(path, "/v1.55") + } + return path +} diff --git a/internal/worktree/worktree.go b/internal/worktree/worktree.go index a99fb8390..0ed3174ec 100644 --- a/internal/worktree/worktree.go +++ b/internal/worktree/worktree.go @@ -6,11 +6,15 @@ package worktree import ( "context" + "errors" "fmt" + "net/url" "os" "os/exec" "path/filepath" "strings" + "sync" + "time" ) type Manager struct { @@ -24,6 +28,8 @@ type Manager struct { // BaseURL overrides the forge host. Set from config [forge].host. // Empty falls back to https://github.com. BaseURL string + + cacheMu sync.Mutex } // Dir is the worktree path for a task. @@ -70,6 +76,25 @@ func (m *Manager) git(ctx context.Context, dir string, args ...string) (string, // If the worktree is already prepared (the daemon may have done this // before container acquisition), it returns the existing directory. func (m *Manager) Prepare(ctx context.Context, owner, repo, base string, issue int, title, body, labels string) (dir, branch string, err error) { + return m.prepare(ctx, owner, repo, base, issue, title, body, labels, "") +} + +// PreparePersistent creates an isolated task worktree using a per-repo bare +// Git cache as an object reference. The cache avoids repeated full network +// clones while --dissociate keeps each task worktree independent so expiry +// cannot break active or parked tasks. +func (m *Manager) PreparePersistent(ctx context.Context, owner, repo, base string, issue int, title, body, labels string, ttl time.Duration) (dir, branch string, err error) { + m.cacheMu.Lock() + defer m.cacheMu.Unlock() + + cache, err := m.prepareCache(ctx, owner, repo, ttl) + if err != nil { + return "", "", err + } + return m.prepare(ctx, owner, repo, base, issue, title, body, labels, cache) +} + +func (m *Manager) prepare(ctx context.Context, owner, repo, base string, issue int, title, body, labels, cache string) (dir, branch string, err error) { dir = m.Dir(owner, repo, issue) branch = archieBranch(issue, title, body, labels) @@ -93,7 +118,12 @@ func (m *Manager) Prepare(ctx context.Context, owner, repo, base string, issue i return "", "", err } url := m.cloneURL(owner, repo) - if _, err := m.git(ctx, filepath.Dir(dir), "clone", "--branch", base, url, dir); err != nil { + cloneArgs := []string{"clone", "--branch", base} + if cache != "" { + cloneArgs = append(cloneArgs, "--reference-if-able", cache, "--dissociate") + } + cloneArgs = append(cloneArgs, url, dir) + if _, err := m.git(ctx, filepath.Dir(dir), cloneArgs...); err != nil { return "", "", err } for _, args := range [][]string{ @@ -108,6 +138,92 @@ func (m *Manager) Prepare(ctx context.Context, owner, repo, base string, issue i return dir, branch, nil } +func (m *Manager) cacheDir(owner, repo string) string { + key := url.PathEscape(owner) + "-" + url.PathEscape(repo) + ".git" + return filepath.Join(m.WorkDir, ".repo-cache", key) +} + +func (m *Manager) prepareCache(ctx context.Context, owner, repo string, ttl time.Duration) (string, error) { + cache := m.cacheDir(owner, repo) + if st, err := os.Stat(filepath.Join(cache, "HEAD")); err == nil && !st.IsDir() { + if ttl > 0 { + if cacheInfo, statErr := os.Stat(cache); statErr == nil && time.Since(cacheInfo.ModTime()) >= ttl { + if err := os.RemoveAll(cache); err != nil { + return "", fmt.Errorf("expire repo cache: %w", err) + } + } + } + } + + if st, err := os.Stat(filepath.Join(cache, "HEAD")); err == nil && !st.IsDir() { + if _, err := m.git(ctx, cache, "fetch", "--prune", "origin"); err != nil { + return "", fmt.Errorf("refresh repo cache: %w", err) + } + if err := os.Chtimes(cache, time.Now(), time.Now()); err != nil { + return "", fmt.Errorf("touch repo cache: %w", err) + } + return cache, nil + } + + if err := os.RemoveAll(cache); err != nil { + return "", fmt.Errorf("reset repo cache: %w", err) + } + if err := os.MkdirAll(filepath.Dir(cache), 0o755); err != nil { + return "", fmt.Errorf("create repo cache dir: %w", err) + } + if _, err := m.git(ctx, filepath.Dir(cache), "clone", "--mirror", m.cloneURL(owner, repo), cache); err != nil { + return "", fmt.Errorf("create repo cache: %w", err) + } + if err := os.Chtimes(cache, time.Now(), time.Now()); err != nil { + return "", fmt.Errorf("touch repo cache: %w", err) + } + return cache, nil +} + +// CleanupExpiredCaches removes per-repo Git object caches not used within ttl. +// Task worktrees are dissociated clones and remain intact. +func (m *Manager) CleanupExpiredCaches(ttl time.Duration) (int, error) { + if ttl <= 0 { + return 0, nil + } + m.cacheMu.Lock() + defer m.cacheMu.Unlock() + + root := filepath.Join(m.WorkDir, ".repo-cache") + entries, err := os.ReadDir(root) + if os.IsNotExist(err) { + return 0, nil + } + if err != nil { + return 0, fmt.Errorf("list repo caches: %w", err) + } + var ( + removed int + errs []error + ) + now := time.Now() + for _, entry := range entries { + if !entry.IsDir() { + continue + } + path := filepath.Join(root, entry.Name()) + info, err := entry.Info() + if err != nil { + errs = append(errs, fmt.Errorf("stat repo cache %s: %w", entry.Name(), err)) + continue + } + if now.Sub(info.ModTime()) < ttl { + continue + } + if err := os.RemoveAll(path); err != nil { + errs = append(errs, fmt.Errorf("remove repo cache %s: %w", entry.Name(), err)) + continue + } + removed++ + } + return removed, errors.Join(errs...) +} + // archieBranch builds a descriptive branch name. Uses conventional // commit prefix (feat/fix/chore) from the issue title when possible, // falls back to labels, then a plain "archie" prefix. diff --git a/internal/worktree/worktree_test.go b/internal/worktree/worktree_test.go index 0155bdcc1..75c3d2202 100644 --- a/internal/worktree/worktree_test.go +++ b/internal/worktree/worktree_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" "testing" + "time" ) // newLocalRemote creates a bare repo with one commit on main and @@ -136,3 +137,73 @@ func TestPrepareCommitPushRoundTrip(t *testing.T) { t.Fatal("cleanup left the worktree behind") } } + +func TestPreparePersistentReusesRepoCacheAndCleanupExpiresIt(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + ctx := context.Background() + host := newLocalRemote(t, "sam", "todo") + m := &Manager{ + WorkDir: t.TempDir(), + Token: "unused-for-file-remotes", + BotUser: "archie-bot", + BotEmail: "archie-bot@example.com", + BaseURL: "file://" + host, + } + const ttl = 24 * time.Hour + + first, _, err := m.PreparePersistent(ctx, "sam", "todo", "main", 41, "feat: first", "", "", ttl) + if err != nil { + t.Fatalf("first persistent prepare: %v", err) + } + cache := m.cacheDir("sam", "todo") + if st, err := os.Stat(filepath.Join(cache, "HEAD")); err != nil || st.IsDir() { + t.Fatalf("repo cache was not created at %s: %v", cache, err) + } + + second, _, err := m.PreparePersistent(ctx, "sam", "todo", "main", 42, "feat: second", "", "", ttl) + if err != nil { + t.Fatalf("second persistent prepare: %v", err) + } + if first == second { + t.Fatalf("persistent tasks share a mutable worktree: %s", first) + } + for _, dir := range []string{first, second} { + if _, err := os.Stat(filepath.Join(dir, ".git")); err != nil { + t.Fatalf("isolated worktree missing at %s: %v", dir, err) + } + } + + expired := time.Now().Add(-ttl - time.Hour) + if err := os.Chtimes(cache, expired, expired); err != nil { + t.Fatal(err) + } + count, err := m.CleanupExpiredCaches(ttl) + if err != nil { + t.Fatal(err) + } + if count != 1 { + t.Fatalf("CleanupExpiredCaches count = %d, want 1", count) + } + if _, err := os.Stat(cache); !os.IsNotExist(err) { + t.Fatalf("expired repo cache still exists: %v", err) + } + for _, dir := range []string{first, second} { + if _, err := os.Stat(filepath.Join(dir, ".git")); err != nil { + t.Fatalf("cache cleanup removed or broke task worktree %s: %v", dir, err) + } + } +} + +func TestCacheDirCannotEscapeWorkDir(t *testing.T) { + m := &Manager{WorkDir: t.TempDir()} + cache := m.cacheDir("../outside", `..\also-outside`) + rel, err := filepath.Rel(filepath.Join(m.WorkDir, ".repo-cache"), cache) + if err != nil { + t.Fatal(err) + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + t.Fatalf("cache path escaped managed root: %s", cache) + } +}