feat(storage): persist agent project state
This commit is contained in:
1 parent
f76645bc2c
commit
d82b232d1b
7 files changed
+459
-9
No files matched your search
@@ -12,7 +12,9 @@ import (
|
|||||||
"github.com/samcharles93/ai-sdk/agentloop"
|
"github.com/samcharles93/ai-sdk/agentloop"
|
||||||
"github.com/samcharles93/ai-sdk/core"
|
"github.com/samcharles93/ai-sdk/core"
|
||||||
"github.com/samcharles93/ai-sdk/runtime"
|
"github.com/samcharles93/ai-sdk/runtime"
|
||||||
|
"github.com/samcharles93/ai-sdk/toolkit"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/skill"
|
||||||
"github.com/samcharles93/archie-core/internal/skillscript"
|
"github.com/samcharles93/archie-core/internal/skillscript"
|
||||||
"github.com/samcharles93/archie-core/internal/tools"
|
"github.com/samcharles93/archie-core/internal/tools"
|
||||||
)
|
)
|
||||||
@@ -84,6 +86,12 @@ func (r *InProcessRunner) Run(ctx context.Context, workspace string, req Request
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return Result{}, fmt.Errorf("build central tool set: %w", err)
|
return Result{}, fmt.Errorf("build central tool set: %w", err)
|
||||||
}
|
}
|
||||||
|
captureTools := captureToolSet(req.CaptureTools, captures)
|
||||||
|
scriptTools := scriptToolSet(workspace)
|
||||||
|
pluginTools, err := pluginToolSet(req, workspace, centralTools, captureTools, scriptTools)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
res, err := r.run(ctx, agentloop.Config{
|
res, err := r.run(ctx, agentloop.Config{
|
||||||
Runtime: r.runtime,
|
Runtime: r.runtime,
|
||||||
ModelRef: req.Model,
|
ModelRef: req.Model,
|
||||||
@@ -98,8 +106,9 @@ func (r *InProcessRunner) Run(ctx context.Context, workspace string, req Request
|
|||||||
ProtectPaths: protectionMatcher(req.Protection, req.ReadOnly),
|
ProtectPaths: protectionMatcher(req.Protection, req.ReadOnly),
|
||||||
Extra: mergeToolSets(
|
Extra: mergeToolSets(
|
||||||
centralTools,
|
centralTools,
|
||||||
captureToolSet(req.CaptureTools, captures),
|
captureTools,
|
||||||
scriptToolSet(workspace),
|
scriptTools,
|
||||||
|
pluginTools,
|
||||||
),
|
),
|
||||||
Logger: r.logger(req),
|
Logger: r.logger(req),
|
||||||
})
|
})
|
||||||
@@ -131,6 +140,48 @@ func (r *InProcessRunner) Run(ctx context.Context, workspace string, req Request
|
|||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func pluginToolSet(req Request, workspace string, occupied ...core.ToolSet) (core.ToolSet, error) {
|
||||||
|
if req.ReadOnly || len(req.Protection.Suffixes)+len(req.Protection.Globs) > 0 || len(req.Gate.Commands) > 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
const params = `{"type":"object","properties":{"input":{"type":"string"}},"required":["input"]}`
|
||||||
|
builtins := toolkit.NewRegistry()
|
||||||
|
if err := toolkit.RegisterBuiltins(builtins, workspace); err != nil {
|
||||||
|
return nil, fmt.Errorf("register built-in tools for plugin collision check: %w", err)
|
||||||
|
}
|
||||||
|
reserved := map[string]struct{}{"finish": {}, "write_note": {}}
|
||||||
|
for _, name := range builtins.Names() {
|
||||||
|
reserved[name] = struct{}{}
|
||||||
|
}
|
||||||
|
set := make(core.ToolSet, len(req.Plugins))
|
||||||
|
for _, spec := range req.Plugins {
|
||||||
|
if _, exists := reserved[spec.Name]; exists {
|
||||||
|
return nil, fmt.Errorf("plugin tool %q conflicts with an agent-loop tool", spec.Name)
|
||||||
|
}
|
||||||
|
for _, tools := range occupied {
|
||||||
|
if _, exists := tools[spec.Name]; exists {
|
||||||
|
return nil, fmt.Errorf("plugin tool %q conflicts with an existing tool", spec.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
plugin := skill.Plugin{Name: spec.Name, Src: spec.Src}
|
||||||
|
set[spec.Name] = core.NewTool(
|
||||||
|
spec.Name,
|
||||||
|
"Run the project-bundled "+spec.Name+" plugin.",
|
||||||
|
json.RawMessage(params),
|
||||||
|
func(_ context.Context, input string) (string, error) {
|
||||||
|
var args struct {
|
||||||
|
Input string `json:"input"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(input), &args); err != nil {
|
||||||
|
return spec.Name + " rejected: arguments must be a JSON object with an input field", nil //nolint:nilerr // rejection feedback lets the model retry with valid arguments
|
||||||
|
}
|
||||||
|
return plugin.Run(args.Input)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return set, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *InProcessRunner) logger(req Request) *slog.Logger {
|
func (r *InProcessRunner) logger(req Request) *slog.Logger {
|
||||||
if r.log == nil {
|
if r.log == nil {
|
||||||
return slog.New(slog.DiscardHandler)
|
return slog.New(slog.DiscardHandler)
|
||||||
|
|||||||
@@ -15,6 +15,47 @@ import (
|
|||||||
"github.com/samcharles93/archie-core/internal/tools"
|
"github.com/samcharles93/archie-core/internal/tools"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestPluginToolSetExecutesBundledPlugin(t *testing.T) {
|
||||||
|
set, err := pluginToolSet(Request{Plugins: []PluginSpec{{Name: "echo", Src: "package main\nfunc Run(input string) string { return input + \"!\" }"}}}, t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tool := set["echo"]
|
||||||
|
got, err := tool.Execute(context.Background(), `{"input":"hello"}`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got != "hello!" {
|
||||||
|
t.Errorf("plugin output = %q, want hello!", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPluginToolSetExcludedFromConstrainedStages(t *testing.T) {
|
||||||
|
requests := []Request{
|
||||||
|
{ReadOnly: true, Plugins: []PluginSpec{{Name: "unsafe"}}},
|
||||||
|
{Protection: Protection{Suffixes: []string{"_test.go"}}, Plugins: []PluginSpec{{Name: "unsafe"}}},
|
||||||
|
{Gate: Gate{Commands: []Command{{Name: "test"}}}, Plugins: []PluginSpec{{Name: "unsafe"}}},
|
||||||
|
}
|
||||||
|
for _, request := range requests {
|
||||||
|
set, err := pluginToolSet(request, t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(set) != 0 {
|
||||||
|
t.Error("plugin tool exposed in constrained stage")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPluginToolSetRejectsAgentLoopCollisions(t *testing.T) {
|
||||||
|
for _, name := range []string{"write", "finish"} {
|
||||||
|
_, err := pluginToolSet(Request{Plugins: []PluginSpec{{Name: name}}}, t.TempDir())
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("plugin %q did not conflict with agent-loop tool", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestInProcessRunnerMapsRequestAndCapturesOutput(t *testing.T) {
|
func TestInProcessRunnerMapsRequestAndCapturesOutput(t *testing.T) {
|
||||||
runner := &InProcessRunner{
|
runner := &InProcessRunner{
|
||||||
runtime: runtime.NewRuntime(runtime.Config{}),
|
runtime: runtime.NewRuntime(runtime.Config{}),
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
package agentworker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||||
|
"github.com/samcharles93/archie-core/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
type persistentRunner struct {
|
||||||
|
agentexec.Runner
|
||||||
|
enabled bool
|
||||||
|
sessionPath string
|
||||||
|
memoryPath string
|
||||||
|
pluginsDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
type memoryRecord struct {
|
||||||
|
TaskID int64 `json:"task_id"`
|
||||||
|
Attempt int `json:"attempt"`
|
||||||
|
Stage string `json:"stage"`
|
||||||
|
Content string `json:"content"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func readProjectMemory(path string) (string, error) {
|
||||||
|
const maxEntries = 100
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
var entries []string
|
||||||
|
scanner := bufio.NewScanner(f)
|
||||||
|
scanner.Buffer(make([]byte, 64*1024), 8*1024*1024)
|
||||||
|
for scanner.Scan() {
|
||||||
|
var record memoryRecord
|
||||||
|
if err := json.Unmarshal(scanner.Bytes(), &record); err != nil {
|
||||||
|
return "", fmt.Errorf("decode memory record: %w", err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(record.Content) != "" {
|
||||||
|
entries = append(entries, "- "+record.Content)
|
||||||
|
if len(entries) > maxEntries {
|
||||||
|
entries = entries[len(entries)-maxEntries:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return "", fmt.Errorf("scan memory records: %w", err)
|
||||||
|
}
|
||||||
|
if len(entries) == 0 {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
return "Project memory from earlier sessions:\n" + strings.Join(entries, "\n"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r persistentRunner) Run(ctx context.Context, workspace string, request agentexec.Request, report agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||||
|
if !r.enabled {
|
||||||
|
return r.Runner.Run(ctx, workspace, request, report)
|
||||||
|
}
|
||||||
|
sessionPath, memoryPath, pluginsDir := r.sessionPath, r.memoryPath, r.pluginsDir
|
||||||
|
if sessionPath == "" {
|
||||||
|
sessionPath = storage.SessionPath
|
||||||
|
}
|
||||||
|
if memoryPath == "" {
|
||||||
|
memoryPath = storage.MemoryPath
|
||||||
|
}
|
||||||
|
if pluginsDir == "" {
|
||||||
|
pluginsDir = storage.PluginsDir
|
||||||
|
}
|
||||||
|
for i := range request.Plugins {
|
||||||
|
plugin := &request.Plugins[i]
|
||||||
|
filename := plugin.Name + ".go"
|
||||||
|
if err := storage.StagePlugin(pluginsDir, filename, []byte(plugin.Src)); err != nil {
|
||||||
|
return agentexec.Result{}, fmt.Errorf("stage plugin %s: %w", plugin.Name, err)
|
||||||
|
}
|
||||||
|
source, err := os.ReadFile(pluginsDir + "/" + filename)
|
||||||
|
if err != nil {
|
||||||
|
return agentexec.Result{}, fmt.Errorf("read staged plugin %s: %w", plugin.Name, err)
|
||||||
|
}
|
||||||
|
plugin.Src = string(source)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, runErr := r.Runner.Run(ctx, workspace, request, report)
|
||||||
|
if runErr != nil {
|
||||||
|
return result, runErr
|
||||||
|
}
|
||||||
|
if err := result.ValidateFor(request); err != nil {
|
||||||
|
return result, fmt.Errorf("validate persistent result: %w", err)
|
||||||
|
}
|
||||||
|
if err := storage.AppendJSONLine(sessionPath, result); err != nil {
|
||||||
|
return result, fmt.Errorf("append session result: %w", err)
|
||||||
|
}
|
||||||
|
for _, note := range result.AppendedNotes {
|
||||||
|
record := memoryRecord{TaskID: request.TaskID, Attempt: request.Attempt, Stage: request.Stage, Content: note}
|
||||||
|
if err := storage.AppendJSONLine(memoryPath, record); err != nil {
|
||||||
|
return result, fmt.Errorf("append project memory: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
package agentworker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||||
|
)
|
||||||
|
|
||||||
|
type persistentRunnerFunc func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error)
|
||||||
|
|
||||||
|
func (f persistentRunnerFunc) Run(ctx context.Context, workspace string, request agentexec.Request, report agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||||
|
return f(ctx, workspace, request, report)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPersistentRunnerRecordsStageMemoryAndPluginOverride(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pluginsDir := filepath.Join(dir, "plugins")
|
||||||
|
if err := os.MkdirAll(pluginsDir, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(pluginsDir, "check.go"), []byte("override"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
runner := persistentRunner{
|
||||||
|
Runner: persistentRunnerFunc(func(_ context.Context, _ string, request agentexec.Request, _ agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||||
|
if got := request.Plugins[0].Src; got != "override" {
|
||||||
|
t.Fatalf("plugin source = %q, want persistent override", got)
|
||||||
|
}
|
||||||
|
return agentexec.Result{Version: agentexec.ProtocolVersion, TaskID: request.TaskID, Attempt: request.Attempt, Stage: request.Stage, Status: agentexec.StatusPassed, AppendedNotes: []string{"remember me"}}, nil
|
||||||
|
}),
|
||||||
|
enabled: true,
|
||||||
|
sessionPath: filepath.Join(dir, "session.jsonl"),
|
||||||
|
memoryPath: filepath.Join(dir, "memory.jsonl"),
|
||||||
|
pluginsDir: pluginsDir,
|
||||||
|
}
|
||||||
|
_, err := runner.Run(context.Background(), t.TempDir(), agentexec.Request{
|
||||||
|
TaskID: 7, Attempt: 2, Stage: "implement",
|
||||||
|
Plugins: []agentexec.PluginSpec{{Name: "check", Src: "bundled"}},
|
||||||
|
}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, path := range []string{runner.sessionPath, runner.memoryPath} {
|
||||||
|
content, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(string(content), "\n") {
|
||||||
|
t.Errorf("%s is not newline-terminated JSONL", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
memory, err := os.ReadFile(runner.memoryPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(memory), `"content":"remember me"`) {
|
||||||
|
t.Errorf("memory record = %s, want appended note", memory)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadProjectMemoryRestoresEarlierNotes(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "memory.jsonl")
|
||||||
|
if err := os.WriteFile(path, []byte("{\"content\":\"first fact\"}\n{\"content\":\"second fact\"}\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := readProjectMemory(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, fact := range []string{"first fact", "second fact"} {
|
||||||
|
if !strings.Contains(got, fact) {
|
||||||
|
t.Errorf("memory prompt %q missing %q", got, fact)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadProjectMemoryAcceptsLargePersistedNote(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "memory.jsonl")
|
||||||
|
note := strings.Repeat("x", 70*1024)
|
||||||
|
encoded := `{"content":"` + note + `"}` + "\n"
|
||||||
|
if err := os.WriteFile(path, []byte(encoded), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := readProjectMemory(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, note) {
|
||||||
|
t.Error("large persisted note was not restored")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPersistentRunnerRejectsFailedResultBeforePersistence(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
runner := persistentRunner{
|
||||||
|
Runner: persistentRunnerFunc(func(_ context.Context, _ string, _ agentexec.Request, _ agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||||
|
return agentexec.Result{AppendedNotes: []string{"untrusted"}}, os.ErrPermission
|
||||||
|
}),
|
||||||
|
enabled: true,
|
||||||
|
sessionPath: filepath.Join(dir, "session.jsonl"),
|
||||||
|
memoryPath: filepath.Join(dir, "memory.jsonl"),
|
||||||
|
pluginsDir: filepath.Join(dir, "plugins"),
|
||||||
|
}
|
||||||
|
_, err := runner.Run(context.Background(), t.TempDir(), agentexec.Request{}, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected runner error")
|
||||||
|
}
|
||||||
|
for _, path := range []string{runner.sessionPath, runner.memoryPath} {
|
||||||
|
if _, statErr := os.Stat(path); !os.IsNotExist(statErr) {
|
||||||
|
t.Errorf("%s persisted rejected output", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"github.com/samcharles93/archie-core/internal/domain/workflow/skillbuild"
|
"github.com/samcharles93/archie-core/internal/domain/workflow/skillbuild"
|
||||||
"github.com/samcharles93/archie-core/internal/domain/workflow/wfeval"
|
"github.com/samcharles93/archie-core/internal/domain/workflow/wfeval"
|
||||||
"github.com/samcharles93/archie-core/internal/events"
|
"github.com/samcharles93/archie-core/internal/events"
|
||||||
|
"github.com/samcharles93/archie-core/internal/storage"
|
||||||
"github.com/samcharles93/archie-core/internal/store"
|
"github.com/samcharles93/archie-core/internal/store"
|
||||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||||
"github.com/samcharles93/archie-core/internal/worktree"
|
"github.com/samcharles93/archie-core/internal/worktree"
|
||||||
@@ -164,6 +165,7 @@ func runTask(ctx context.Context, req taskrun.Request, dependencies taskDependen
|
|||||||
if agent == nil {
|
if agent == nil {
|
||||||
return nil, fmt.Errorf("no agent runner configured for task %d", req.Task.ID)
|
return nil, fmt.Errorf("no agent runner configured for task %d", req.Task.ID)
|
||||||
}
|
}
|
||||||
|
agent = persistentRunner{Runner: agent, enabled: req.Repo.PersistentStorage}
|
||||||
|
|
||||||
// A workflow run in this process publishes to an in-process *events.Bus
|
// A workflow run in this process publishes to an in-process *events.Bus
|
||||||
// the daemon cannot see -- archied and archie-agent are separate
|
// the daemon cannot see -- archied and archie-agent are separate
|
||||||
@@ -194,6 +196,15 @@ func runTask(ctx context.Context, req taskrun.Request, dependencies taskDependen
|
|||||||
Log: log,
|
Log: log,
|
||||||
CustomStages: wfeval.Discover,
|
CustomStages: wfeval.Discover,
|
||||||
}
|
}
|
||||||
|
if req.Repo.PersistentStorage {
|
||||||
|
memory, readErr := readProjectMemory(storage.MemoryPath)
|
||||||
|
if readErr != nil {
|
||||||
|
return nil, fmt.Errorf("read project memory: %w", readErr)
|
||||||
|
}
|
||||||
|
if memory != "" {
|
||||||
|
tc.SystemPrompt = func() string { return memory }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
workflow.Run(ctx, wf, tc)
|
workflow.Run(ctx, wf, tc)
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,9 @@
|
|||||||
// the agent's own commit cannot sweep it onto the
|
// the agent's own commit cannot sweep it onto the
|
||||||
// task branch: go-git's Add ignores .gitignore.
|
// task branch: go-git's Add ignores .gitignore.
|
||||||
// repo/ -- per-repo persistent volume (optional, on request)
|
// repo/ -- per-repo persistent volume (optional, on request)
|
||||||
|
// session.jsonl -- agent session transcript, appended per stage
|
||||||
|
// memory.jsonl -- cross-session project memory
|
||||||
|
// plugins/ -- daemon-staged bundled plugins
|
||||||
// cache/
|
// cache/
|
||||||
// go/ -- GOMODCACHE (shared across all tasks)
|
// go/ -- GOMODCACHE (shared across all tasks)
|
||||||
// node/ -- npm cache (shared)
|
// node/ -- npm cache (shared)
|
||||||
@@ -19,20 +22,17 @@
|
|||||||
// pip/ -- pip cache (shared)
|
// pip/ -- pip cache (shared)
|
||||||
// cargo/ -- Rust cargo cache (shared)
|
// cargo/ -- Rust cargo cache (shared)
|
||||||
//
|
//
|
||||||
// TODO(PRD §3): session.jsonl, memory.jsonl, plugins/ volume -- these
|
|
||||||
// require agent-side implementation before the storage layer can mount
|
|
||||||
// them. session.jsonl needs per-stage structured output from the agent;
|
|
||||||
// memory.jsonl needs cross-session persistence; plugins/ needs daemon-
|
|
||||||
// staged bundled plugins.
|
|
||||||
//
|
|
||||||
// Cache volumes are named Docker volumes created once and shared across all
|
// Cache volumes are named Docker volumes created once and shared across all
|
||||||
// tasks. They are never deleted -- the daemon operator manages them.
|
// tasks. They are never deleted -- the daemon operator manages them.
|
||||||
package storage
|
package storage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -65,12 +65,75 @@ const (
|
|||||||
MountTypeVolume = "volume"
|
MountTypeVolume = "volume"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// AppendJSONLine appends one complete JSON value to a JSONL file. The value is
|
||||||
|
// marshalled before opening the file so a marshal failure cannot leave a
|
||||||
|
// partial record behind.
|
||||||
|
func AppendJSONLine(path string, value any) error {
|
||||||
|
record, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal JSONL record: %w", err)
|
||||||
|
}
|
||||||
|
record = append(record, '\n')
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
return fmt.Errorf("create JSONL directory: %w", err)
|
||||||
|
}
|
||||||
|
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("open JSONL file: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := f.Write(record); err != nil {
|
||||||
|
_ = f.Close()
|
||||||
|
return fmt.Errorf("append JSONL record: %w", err)
|
||||||
|
}
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
return fmt.Errorf("close JSONL file: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// StagePlugin writes a bundled plugin unless persistent project storage
|
||||||
|
// already contains a plugin with the same filename. Persistent copies are
|
||||||
|
// operator-owned overrides and therefore win over daemon-staged source.
|
||||||
|
func StagePlugin(dir, name string, source []byte) error {
|
||||||
|
if filepath.Base(name) != name || name == "." || name == "" {
|
||||||
|
return fmt.Errorf("invalid plugin filename %q", name)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
return fmt.Errorf("create plugin directory: %w", err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(dir, name)
|
||||||
|
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||||
|
if errors.Is(err, os.ErrExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create staged plugin: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := f.Write(source); err != nil {
|
||||||
|
_ = f.Close()
|
||||||
|
return fmt.Errorf("write staged plugin: %w", err)
|
||||||
|
}
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
return fmt.Errorf("close staged plugin: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// WorktreeMountDir is the fixed container path where a task's worktree is
|
// WorktreeMountDir is the fixed container path where a task's worktree is
|
||||||
// bind-mounted. archie-agent uses this path directly rather than the host
|
// bind-mounted. archie-agent uses this path directly rather than the host
|
||||||
// path archied's worktree.Manager reports -- the two processes see the same
|
// path archied's worktree.Manager reports -- the two processes see the same
|
||||||
// files at different paths.
|
// files at different paths.
|
||||||
const WorktreeMountDir = "/data/worktree"
|
const WorktreeMountDir = "/data/worktree"
|
||||||
|
|
||||||
|
// Persistent storage paths share the per-repository volume so session output,
|
||||||
|
// project memory, and staged plugins survive individual task containers.
|
||||||
|
const (
|
||||||
|
PersistentMountDir = "/data/repo"
|
||||||
|
SessionPath = PersistentMountDir + "/session.jsonl"
|
||||||
|
MemoryPath = PersistentMountDir + "/memory.jsonl"
|
||||||
|
PluginsDir = PersistentMountDir + "/plugins"
|
||||||
|
)
|
||||||
|
|
||||||
// Backend prepares container storage. Each container runtime backend
|
// Backend prepares container storage. Each container runtime backend
|
||||||
// (Docker, future containerd, etc.) implements this interface.
|
// (Docker, future containerd, etc.) implements this interface.
|
||||||
type Backend interface {
|
type Backend interface {
|
||||||
@@ -237,7 +300,7 @@ func (d *DockerBackend) Setup(ctx context.Context, task TaskRef) ([]Mount, error
|
|||||||
mounts = append(mounts, Mount{
|
mounts = append(mounts, Mount{
|
||||||
Type: MountTypeVolume,
|
Type: MountTypeVolume,
|
||||||
Source: volName,
|
Source: volName,
|
||||||
Destination: "/data/repo",
|
Destination: PersistentMountDir,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -324,6 +325,60 @@ func TestDockerBackendSetupWithPersistentStorage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPersistentStoragePaths(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
got string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "root", got: PersistentMountDir, want: "/data/repo"},
|
||||||
|
{name: "session", got: SessionPath, want: "/data/repo/session.jsonl"},
|
||||||
|
{name: "memory", got: MemoryPath, want: "/data/repo/memory.jsonl"},
|
||||||
|
{name: "plugins", got: PluginsDir, want: "/data/repo/plugins"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if tt.got != tt.want {
|
||||||
|
t.Errorf("path = %q, want %q", tt.got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAppendJSONLineAppendsCompleteRecords(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "nested", "session.jsonl")
|
||||||
|
for _, value := range []map[string]int{{"stage": 1}, {"stage": 2}} {
|
||||||
|
if err := AppendJSONLine(path, value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if want := "{\"stage\":1}\n{\"stage\":2}\n"; string(got) != want {
|
||||||
|
t.Errorf("JSONL = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStagePluginPreservesPersistentOverride(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := StagePlugin(dir, "check.go", []byte("bundled")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := StagePlugin(dir, "check.go", []byte("replacement")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := os.ReadFile(filepath.Join(dir, "check.go"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(got) != "bundled" {
|
||||||
|
t.Errorf("plugin = %q, want persistent override", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDockerBackendSetupWithoutPersistentStorage(t *testing.T) {
|
func TestDockerBackendSetupWithoutPersistentStorage(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
|||||||
Reference in new issue
Block a user