fix(agentworker): move JSONL decode into storage, unblocking CI
TestApplicationOwnsNoTransportWireMechanics was failing on main since
d82b232: persistent_runner.go imported encoding/json to decode its
own project-memory JSONL log, which the architecture test treats as
a transport-wire-mechanics violation. That ban exists to keep NATS
message encoding out of the app layer -- this was local disk I/O, not
wire traffic, but the test can't tell the difference from an import
alone, and it shouldn't have to: the fix is to not decode JSON in
agentworker at all.
storage.ReadJSONLines is AppendJSONLine's read-side counterpart,
added to the package that already owns this format (see
organisation.md, "a package owns its own format end to end").
persistent_runner.go now calls it instead of scanning and decoding
itself, and no longer imports encoding/json or bufio.
This was blocking every deploy.yml run on main, independent of and
found while shipping the video-generation PR (#561).
This commit is contained in:
1 parent
e04045907c
commit
27ea40fec5
3 files changed
+117
-22
No files matched your search
@@ -1,10 +1,7 @@
|
||||
package agentworker
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -30,32 +27,18 @@ type memoryRecord struct {
|
||||
|
||||
func readProjectMemory(path string) (string, error) {
|
||||
const maxEntries = 100
|
||||
f, err := os.Open(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var entries []string
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 64*1024), 8*1024*1024)
|
||||
for scanner.Scan() {
|
||||
var record memoryRecord
|
||||
if err := json.Unmarshal(scanner.Bytes(), &record); err != nil {
|
||||
return "", fmt.Errorf("decode memory record: %w", err)
|
||||
}
|
||||
err := storage.ReadJSONLines(path, func(record memoryRecord) error {
|
||||
if strings.TrimSpace(record.Content) != "" {
|
||||
entries = append(entries, "- "+record.Content)
|
||||
if len(entries) > maxEntries {
|
||||
entries = entries[len(entries)-maxEntries:]
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return "", fmt.Errorf("scan memory records: %w", err)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read project memory: %w", err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return "", nil
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -91,6 +92,42 @@ func AppendJSONLine(path string, value any) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadJSONLines decodes each line of the JSONL file at path as a T, calling
|
||||
// fn for each. A path that does not exist yet is not an error -- no log
|
||||
// written and an empty log mean the same thing to a caller -- so fn is
|
||||
// simply never called. A malformed line, or an error fn itself returns,
|
||||
// stops the scan and is returned wrapped.
|
||||
//
|
||||
// This is AppendJSONLine's read-side counterpart, kept in the package that
|
||||
// owns the format: see organisation.md's "a package owns its own format end
|
||||
// to end."
|
||||
func ReadJSONLines[T any](path string, fn func(T) error) error {
|
||||
f, err := os.Open(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("open JSONL file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 64*1024), 8*1024*1024)
|
||||
for scanner.Scan() {
|
||||
var record T
|
||||
if err := json.Unmarshal(scanner.Bytes(), &record); err != nil {
|
||||
return fmt.Errorf("decode JSONL record: %w", err)
|
||||
}
|
||||
if err := fn(record); err != nil {
|
||||
return fmt.Errorf("handle JSONL record: %w", err)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return fmt.Errorf("scan JSONL file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StagePlugin writes a bundled plugin unless persistent project storage
|
||||
// already contains a plugin with the same filename. Persistent copies are
|
||||
// operator-owned overrides and therefore win over daemon-staged source.
|
||||
|
||||
@@ -3,6 +3,7 @@ package storage
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"go/format"
|
||||
"io"
|
||||
@@ -362,6 +363,80 @@ func TestAppendJSONLineAppendsCompleteRecords(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type jsonlTestRecord struct {
|
||||
Stage int `json:"stage"`
|
||||
}
|
||||
|
||||
// The reader is AppendJSONLine's counterpart: a value written by one must
|
||||
// come back unchanged through the other, in the order written.
|
||||
func TestReadJSONLinesRoundTripsAppendJSONLine(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "session.jsonl")
|
||||
for _, v := range []jsonlTestRecord{{Stage: 1}, {Stage: 2}, {Stage: 3}} {
|
||||
if err := AppendJSONLine(path, v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var got []jsonlTestRecord
|
||||
if err := ReadJSONLines(path, func(v jsonlTestRecord) error {
|
||||
got = append(got, v)
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := []jsonlTestRecord{{Stage: 1}, {Stage: 2}, {Stage: 3}}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("records = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("records = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A log that has never been written is not an error: no memory yet and no
|
||||
// file yet mean the same thing to a caller.
|
||||
func TestReadJSONLinesMissingFileIsNotAnError(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "never-written.jsonl")
|
||||
called := false
|
||||
if err := ReadJSONLines(path, func(jsonlTestRecord) error { called = true; return nil }); err != nil {
|
||||
t.Fatalf("ReadJSONLines: %v", err)
|
||||
}
|
||||
if called {
|
||||
t.Error("fn was called for a file that does not exist")
|
||||
}
|
||||
}
|
||||
|
||||
// A malformed line must fail loudly rather than silently skip a record --
|
||||
// project memory is meant to be trusted context for later sessions, so a
|
||||
// corrupt entry must not vanish unnoticed.
|
||||
func TestReadJSONLinesMalformedLineIsAnError(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "session.jsonl")
|
||||
if err := os.WriteFile(path, []byte("not json\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ReadJSONLines(path, func(jsonlTestRecord) error { return nil }); err == nil {
|
||||
t.Fatal("expected an error for a malformed line")
|
||||
}
|
||||
}
|
||||
|
||||
// fn's own error must stop the scan and propagate, so a caller can bail out
|
||||
// early (e.g. once it has enough records) without ReadJSONLines masking a
|
||||
// real failure as success.
|
||||
func TestReadJSONLinesPropagatesCallbackError(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "session.jsonl")
|
||||
if err := AppendJSONLine(path, jsonlTestRecord{Stage: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sentinel := errors.New("stop here")
|
||||
err := ReadJSONLines(path, func(jsonlTestRecord) error { return sentinel })
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("err = %v, want it to wrap the callback's own error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePluginPreservesPersistentOverride(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := StagePlugin(dir, "check.go", []byte("bundled")); err != nil {
|
||||
|
||||
Reference in new issue
Block a user