- Reword docs and comments to drop vague "legacy" qualifiers where they no longer describe deprecation status - Remove unused code: RecentEvents, MessageEvent.ToLegacy, index setState, stderr redaction tool, telegramOnlyCommands - Delete or comment out tests covering removed or superseded behavior
559 lines
18 KiB
Go
559 lines
18 KiB
Go
// Package indexing provides language-neutral workspace text indexing.
|
|
package indexing
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"regexp/syntax"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/codesearch/index"
|
|
"github.com/google/uuid"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
const (
|
|
// MaxCandidateFiles bounds helper output and grep argv construction.
|
|
MaxCandidateFiles = 2000
|
|
maxCandidateArgBytes = 24 * 1024
|
|
buildTimeout = 10 * time.Minute
|
|
metadataTimeout = 5 * time.Second
|
|
)
|
|
|
|
// CandidateProvider returns files which may match a grep query. A false
|
|
// handled result tells the caller to use its normal unindexed search path.
|
|
type CandidateProvider interface {
|
|
Candidates(context.Context, string, bool, bool) ([]string, bool)
|
|
}
|
|
|
|
// Manager owns the workspace index lifecycle. The codesearch implementation
|
|
// runs in a child process so upstream fatal exits and mmap lifetime cannot
|
|
// terminate or leak inside the interactive agent process.
|
|
type Manager struct {
|
|
root string
|
|
indexPath string
|
|
dbPath string
|
|
runner helperRunner
|
|
log *slog.Logger
|
|
}
|
|
|
|
// logger is nil-safe: a Manager may be constructed as a struct literal
|
|
// (the tests do), so defaulting at the point of use rather than in the
|
|
// constructor is what actually holds.
|
|
func (m *Manager) logger() *slog.Logger {
|
|
if m == nil || m.log == nil {
|
|
return slog.Default()
|
|
}
|
|
return m.log
|
|
}
|
|
|
|
type helperRunner interface {
|
|
Build(context.Context, string, string) error
|
|
Candidates(context.Context, string, string, bool, bool) ([]string, error)
|
|
}
|
|
|
|
type processRunner struct{}
|
|
|
|
// Config locates the index storage for a Manager.
|
|
//
|
|
// Archied resolves these in its config layer ([indexing] in config.toml,
|
|
// defaulted off work_dir by config.finalize) and passes them down. Defaulting
|
|
// deliberately does NOT live here so this package stays free of archie-core's
|
|
// config.
|
|
type Config struct {
|
|
// IndexDir holds the mmap-friendly codesearch sidecar files.
|
|
IndexDir string
|
|
// DBPath is the SQLite database holding index lifecycle metadata.
|
|
DBPath string
|
|
// Log receives index build failures. Without it a failed build is
|
|
// invisible: the manager degrades to unindexed search and the reason
|
|
// is only ever written to a database column nobody reads.
|
|
Log *slog.Logger
|
|
}
|
|
|
|
// NewManager prepares lifecycle storage and starts an asynchronous atomic
|
|
// refresh. An existing index remains available while the refresh runs.
|
|
func NewManager(ctx context.Context, cfg Config, root string) (*Manager, error) {
|
|
if strings.TrimSpace(cfg.IndexDir) == "" {
|
|
return nil, fmt.Errorf("indexing: IndexDir is required")
|
|
}
|
|
if strings.TrimSpace(cfg.DBPath) == "" {
|
|
return nil, fmt.Errorf("indexing: DBPath is required")
|
|
}
|
|
absRoot, err := filepath.Abs(root)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("resolve workspace root: %w", err)
|
|
}
|
|
if err := os.MkdirAll(cfg.IndexDir, 0o700); err != nil {
|
|
return nil, fmt.Errorf("create workspace index directory: %w", err)
|
|
}
|
|
m := &Manager{
|
|
root: filepath.Clean(absRoot),
|
|
indexPath: filepath.Join(cfg.IndexDir, workspaceKey(absRoot)+".csearch"),
|
|
dbPath: cfg.DBPath,
|
|
runner: processRunner{},
|
|
log: cfg.Log,
|
|
}
|
|
if err := m.ensureSchema(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
m.refreshAsync(ctx)
|
|
return m, nil
|
|
}
|
|
|
|
// Candidates implements the tools.GrepIndex contract.
|
|
func (m *Manager) Candidates(ctx context.Context, pattern string, literal, caseSensitive bool) ([]string, bool) {
|
|
state, err := m.state(ctx)
|
|
if err != nil || state.IndexedAt.IsZero() {
|
|
return nil, false
|
|
}
|
|
if _, err := os.Stat(state.IndexPath); err != nil {
|
|
return nil, false
|
|
}
|
|
files, err := m.runner.Candidates(ctx, state.IndexPath, pattern, literal, caseSensitive)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
|
|
// The sidecar is a snapshot. Include files changed or created since the
|
|
// build began so indexed search cannot miss live workspace edits.
|
|
merged, ok := m.mergeCandidateFiles(ctx, files, state)
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
slices.Sort(merged)
|
|
return merged, true
|
|
}
|
|
|
|
// mergeCandidateFiles combines sidecar results with workspace files that
|
|
// were changed or created since the index was built.
|
|
func (m *Manager) mergeCandidateFiles(ctx context.Context, files []string, state indexState) ([]string, bool) {
|
|
workspaceFiles, err := WorkspaceFiles(ctx, m.root)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
changedFiles := changedGitFiles(ctx, m.root)
|
|
seen := make(map[string]struct{}, len(files))
|
|
merged := make([]string, 0, len(files))
|
|
argumentBytes := 0
|
|
add := func(path string) {
|
|
path, ok := confinedWorkspaceFile(m.root, path)
|
|
if !ok {
|
|
return
|
|
}
|
|
if _, ok := seen[path]; ok {
|
|
return
|
|
}
|
|
seen[path] = struct{}{}
|
|
merged = append(merged, path)
|
|
argumentBytes += len(path) + 1
|
|
}
|
|
for _, path := range files {
|
|
add(path)
|
|
}
|
|
for _, path := range workspaceFiles {
|
|
info, statErr := os.Stat(path)
|
|
_, changed := changedFiles[path]
|
|
if statErr == nil && (changed || !info.ModTime().Before(state.IndexedAt)) {
|
|
add(path)
|
|
}
|
|
}
|
|
if len(merged) == 0 || len(merged) > MaxCandidateFiles || argumentBytes > maxCandidateArgBytes {
|
|
return nil, false
|
|
}
|
|
return merged, true
|
|
}
|
|
|
|
func changedGitFiles(ctx context.Context, root string) map[string]struct{} {
|
|
changed := map[string]struct{}{}
|
|
commands := [][]string{
|
|
{"-C", root, "ls-files", "-m", "-o", "--exclude-standard", "-z"},
|
|
{"-C", root, "diff", "--cached", "--name-only", "--diff-filter=ACMR", "-z"},
|
|
}
|
|
for _, args := range commands {
|
|
out, err := exec.CommandContext(ctx, "git", args...).Output()
|
|
if err != nil {
|
|
return map[string]struct{}{}
|
|
}
|
|
for name := range strings.SplitSeq(string(out), "\x00") {
|
|
if name != "" {
|
|
path := filepath.Join(root, filepath.FromSlash(name))
|
|
if confined, ok := confinedWorkspaceFile(root, path); ok {
|
|
changed[confined] = struct{}{}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return changed
|
|
}
|
|
|
|
type indexState struct {
|
|
IndexPath string
|
|
IndexedAt time.Time
|
|
}
|
|
|
|
func (m *Manager) refreshAsync(parent context.Context) {
|
|
go func() {
|
|
ctx, cancel := context.WithTimeout(parent, buildTimeout)
|
|
defer cancel()
|
|
startedAt := time.Now().UTC()
|
|
buildID := uuid.NewString()
|
|
previous, _ := m.state(ctx)
|
|
claimed, err := m.claimBuild(ctx, startedAt, buildID)
|
|
if err != nil {
|
|
// Degrading to unindexed search is safe but slow, and until
|
|
// this is logged the only record of why is a database column
|
|
// with no reader.
|
|
m.logger().Warn("workspace index build not claimed", "root", m.root, "error", err)
|
|
return
|
|
}
|
|
if !claimed {
|
|
// Another process holds the build claim; not an error.
|
|
return
|
|
}
|
|
finalPath := m.indexPath + "." + buildID
|
|
tmp := finalPath + ".tmp"
|
|
_ = os.Remove(tmp)
|
|
if err := m.runner.Build(ctx, m.root, tmp); err != nil {
|
|
_ = os.Remove(tmp)
|
|
m.logger().Warn("workspace index build failed; falling back to unindexed search",
|
|
"root", m.root, "error", err)
|
|
m.finishBuild(parent, buildID, "error", time.Time{}, "", err.Error())
|
|
return
|
|
}
|
|
if err := os.Rename(tmp, finalPath); err != nil {
|
|
_ = os.Remove(tmp)
|
|
m.logger().Warn("workspace index could not be published",
|
|
"root", m.root, "error", err)
|
|
m.finishBuild(parent, buildID, "error", time.Time{}, "", err.Error())
|
|
return
|
|
}
|
|
if !m.finishBuild(parent, buildID, "ready", startedAt, finalPath, "") {
|
|
_ = os.Remove(finalPath)
|
|
m.logger().Warn("workspace index build superseded before publication", "root", m.root)
|
|
return
|
|
}
|
|
m.logger().Info("workspace index ready", "root", m.root, "index", finalPath)
|
|
if previous.IndexPath != "" && previous.IndexPath != finalPath &&
|
|
(previous.IndexPath == m.indexPath || strings.HasPrefix(previous.IndexPath, m.indexPath+".")) {
|
|
_ = os.Remove(previous.IndexPath)
|
|
}
|
|
}()
|
|
}
|
|
|
|
func (m *Manager) finishBuild(parent context.Context, buildID, status string, indexedAt time.Time, indexPath, lastError string) bool {
|
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(parent), metadataTimeout)
|
|
defer cancel()
|
|
db, err := openIndexDB(m.dbPath)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
// Each helper opens its own single-connection handle and the write has
|
|
// already been committed by the time this runs, so a Close error carries
|
|
// no recoverable information.
|
|
defer func() { _ = db.Close() }()
|
|
indexed := ""
|
|
if !indexedAt.IsZero() {
|
|
indexed = indexedAt.Format(time.RFC3339Nano)
|
|
}
|
|
result, err := db.ExecContext(ctx, `UPDATE workspace_indexes SET
|
|
index_path=CASE WHEN ? = 'ready' THEN ? ELSE index_path END,
|
|
status=?, indexed_at=CASE WHEN ? = 'ready' THEN ? ELSE indexed_at END,
|
|
last_error=?, updated_at=?, generation=generation + CASE WHEN ? = 'ready' THEN 1 ELSE 0 END
|
|
WHERE root=? AND build_id=?`, status, indexPath, status, status, indexed,
|
|
lastError, time.Now().UTC().Format(time.RFC3339Nano), status, m.root, buildID)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
rows, err := result.RowsAffected()
|
|
return err == nil && rows == 1
|
|
}
|
|
|
|
func (m *Manager) claimBuild(ctx context.Context, startedAt time.Time, buildID string) (bool, error) {
|
|
db, err := openIndexDB(m.dbPath)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
// Each helper opens its own single-connection handle and the write has
|
|
// already been committed by the time this runs, so a Close error carries
|
|
// no recoverable information.
|
|
defer func() { _ = db.Close() }()
|
|
cutoff := startedAt.Add(-buildTimeout).Format(time.RFC3339Nano)
|
|
result, err := db.ExecContext(ctx, `INSERT INTO workspace_indexes (root, index_path, status, indexed_at, build_started_at, build_id, last_error, updated_at)
|
|
VALUES (?, ?, 'building', '', ?, ?, '', ?)
|
|
ON CONFLICT(root) DO UPDATE SET status='building',
|
|
build_started_at=excluded.build_started_at, build_id=excluded.build_id, last_error='', updated_at=excluded.updated_at
|
|
WHERE workspace_indexes.status != 'building' OR workspace_indexes.updated_at < ?`,
|
|
m.root, m.indexPath, startedAt.Format(time.RFC3339Nano), buildID, startedAt.Format(time.RFC3339Nano), cutoff)
|
|
if err != nil {
|
|
return false, fmt.Errorf("claim workspace index build: %w", err)
|
|
}
|
|
rows, err := result.RowsAffected()
|
|
if err != nil {
|
|
return false, fmt.Errorf("inspect workspace index build claim: %w", err)
|
|
}
|
|
return rows > 0, nil
|
|
}
|
|
|
|
func (m *Manager) ensureSchema(ctx context.Context) error {
|
|
db, err := openIndexDB(m.dbPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Each helper opens its own single-connection handle and the write has
|
|
// already been committed by the time this runs, so a Close error carries
|
|
// no recoverable information.
|
|
defer func() { _ = db.Close() }()
|
|
_, err = db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS workspace_indexes (
|
|
root TEXT PRIMARY KEY,
|
|
index_path TEXT NOT NULL,
|
|
status TEXT NOT NULL,
|
|
format_version TEXT NOT NULL DEFAULT 'codesearch-v2',
|
|
generation INTEGER NOT NULL DEFAULT 0,
|
|
indexed_at TEXT NOT NULL DEFAULT '',
|
|
build_started_at TEXT NOT NULL DEFAULT '',
|
|
build_id TEXT NOT NULL DEFAULT '',
|
|
last_error TEXT NOT NULL DEFAULT '',
|
|
updated_at TEXT NOT NULL
|
|
)`)
|
|
if err != nil {
|
|
return fmt.Errorf("create workspace index metadata: %w", err)
|
|
}
|
|
for _, migration := range []string{
|
|
`ALTER TABLE workspace_indexes ADD COLUMN format_version TEXT NOT NULL DEFAULT 'codesearch-v2'`,
|
|
`ALTER TABLE workspace_indexes ADD COLUMN generation INTEGER NOT NULL DEFAULT 0`,
|
|
`ALTER TABLE workspace_indexes ADD COLUMN build_started_at TEXT NOT NULL DEFAULT ''`,
|
|
`ALTER TABLE workspace_indexes ADD COLUMN build_id TEXT NOT NULL DEFAULT ''`,
|
|
} {
|
|
if _, migrationErr := db.ExecContext(ctx, migration); migrationErr != nil && !strings.Contains(migrationErr.Error(), "duplicate column name") {
|
|
return fmt.Errorf("migrate workspace index metadata: %w", migrationErr)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *Manager) state(ctx context.Context) (indexState, error) {
|
|
db, err := openIndexDB(m.dbPath)
|
|
if err != nil {
|
|
return indexState{}, err
|
|
}
|
|
// Each helper opens its own single-connection handle and the write has
|
|
// already been committed by the time this runs, so a Close error carries
|
|
// no recoverable information.
|
|
defer func() { _ = db.Close() }()
|
|
var indexed, indexPath string
|
|
err = db.QueryRowContext(ctx, `SELECT index_path, indexed_at FROM workspace_indexes WHERE root = ?`, m.root).Scan(&indexPath, &indexed)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return indexState{}, nil
|
|
}
|
|
if err != nil {
|
|
return indexState{}, fmt.Errorf("read workspace index metadata: %w", err)
|
|
}
|
|
at, _ := time.Parse(time.RFC3339Nano, indexed)
|
|
return indexState{IndexPath: indexPath, IndexedAt: at}, nil
|
|
}
|
|
|
|
func openIndexDB(path string) (*sql.DB, error) {
|
|
// modernc.org/sqlite only applies pragmas passed via repeated _pragma
|
|
// params; unrecognized keys like _journal_mode/_busy_timeout are
|
|
// silently ignored, which previously left the busy timeout at 0 and
|
|
// caused SQLITE_BUSY on any lock contention instead of a bounded wait.
|
|
db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open workspace index metadata: %w", err)
|
|
}
|
|
db.SetMaxOpenConns(1)
|
|
return db, nil
|
|
}
|
|
|
|
func workspaceKey(root string) string {
|
|
sum := sha256.Sum256([]byte(filepath.Clean(root)))
|
|
return hex.EncodeToString(sum[:12])
|
|
}
|
|
|
|
func (processRunner) Build(ctx context.Context, root, target string) error {
|
|
_, err := runHelper(ctx, "build", "--root", root, "--index", target)
|
|
return err
|
|
}
|
|
|
|
func (processRunner) Candidates(ctx context.Context, indexPath, pattern string, literal, caseSensitive bool) ([]string, error) {
|
|
args := []string{"candidates", "--index", indexPath, "--pattern", pattern}
|
|
if literal {
|
|
args = append(args, "--literal")
|
|
}
|
|
if caseSensitive {
|
|
args = append(args, "--case-sensitive")
|
|
}
|
|
out, err := runHelper(ctx, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var files []string
|
|
if err := json.Unmarshal(out, &files); err != nil {
|
|
return nil, fmt.Errorf("decode codesearch candidates: %w", err)
|
|
}
|
|
return files, nil
|
|
}
|
|
|
|
func runHelper(ctx context.Context, args ...string) ([]byte, error) {
|
|
executable, err := os.Executable()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("locate archied executable: %w", err)
|
|
}
|
|
cmd := exec.CommandContext(ctx, executable, append([]string{"workspace-codesearch"}, args...)...)
|
|
out, err := cmd.Output()
|
|
if err != nil {
|
|
var exitErr *exec.ExitError
|
|
if errors.As(err, &exitErr) {
|
|
return nil, fmt.Errorf("codesearch helper: %s", strings.TrimSpace(string(exitErr.Stderr)))
|
|
}
|
|
return nil, fmt.Errorf("run codesearch helper: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// BuildIndex creates a codesearch sidecar for root. It is called only inside
|
|
// the isolated helper process.
|
|
func BuildIndex(ctx context.Context, root, target string) error {
|
|
files, err := WorkspaceFiles(ctx, root)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
w := index.Create(target)
|
|
w.AddRoots([]index.Path{index.MakePath(root)})
|
|
for _, path := range files {
|
|
if err := w.AddFile(path); err != nil {
|
|
continue
|
|
}
|
|
}
|
|
w.Flush()
|
|
return nil
|
|
}
|
|
|
|
// IndexCandidates returns the conservative trigram candidate set. It is
|
|
// called only inside the isolated helper process.
|
|
func IndexCandidates(indexPath, pattern string, literal, caseSensitive bool) (files []string, err error) {
|
|
defer func() {
|
|
if recovered := recover(); recovered != nil {
|
|
files = nil
|
|
err = fmt.Errorf("open codesearch index: %v", recovered)
|
|
}
|
|
}()
|
|
if literal {
|
|
pattern = regexp.QuoteMeta(pattern)
|
|
}
|
|
if !caseSensitive && !hasUppercase(pattern) {
|
|
pattern = "(?i:" + pattern + ")"
|
|
}
|
|
re, err := syntax.Parse(pattern, syntax.Perl)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parse search pattern: %w", err)
|
|
}
|
|
ix := index.Open(indexPath)
|
|
ids := ix.PostingQuery(index.RegexpQuery(re))
|
|
files = make([]string, 0, len(ids))
|
|
for _, id := range ids {
|
|
files = append(files, ix.Name(id).String())
|
|
}
|
|
return files, nil
|
|
}
|
|
|
|
// WorkspaceFiles lists searchable workspace files, preferring Git's ignore
|
|
// semantics and falling back to a conservative filesystem walk.
|
|
func WorkspaceFiles(ctx context.Context, root string) ([]string, error) {
|
|
cmd := exec.CommandContext(ctx, "git", "-C", root, "ls-files", "-co", "--exclude-standard", "-z")
|
|
out, err := cmd.Output()
|
|
if err == nil {
|
|
parts := strings.Split(string(out), "\x00")
|
|
files := make([]string, 0, len(parts))
|
|
for _, part := range parts {
|
|
if part != "" {
|
|
path := filepath.Join(root, filepath.FromSlash(part))
|
|
if confined, ok := confinedWorkspaceFile(root, path); ok {
|
|
files = append(files, confined)
|
|
}
|
|
}
|
|
}
|
|
slices.Sort(files)
|
|
return files, nil
|
|
}
|
|
|
|
var files []string
|
|
err = filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return nil //nolint:nilerr // an unreadable entry skips that path and continues the walk; a best-effort index must not abort on one bad file
|
|
}
|
|
if path != root && strings.HasPrefix(entry.Name(), ".") {
|
|
if entry.IsDir() {
|
|
return filepath.SkipDir
|
|
}
|
|
return nil
|
|
}
|
|
if !entry.IsDir() && entry.Type().IsRegular() {
|
|
if confined, ok := confinedWorkspaceFile(root, path); ok {
|
|
files = append(files, confined)
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("walk workspace files: %w", err)
|
|
}
|
|
slices.Sort(files)
|
|
return files, nil
|
|
}
|
|
|
|
func confinedWorkspaceFile(root, path string) (string, bool) {
|
|
absRoot, err := filepath.Abs(root)
|
|
if err != nil {
|
|
return "", false
|
|
}
|
|
absPath, err := filepath.Abs(path)
|
|
if err != nil {
|
|
return "", false
|
|
}
|
|
// Reject a symlink at the final component before resolving parent links.
|
|
info, err := os.Lstat(absPath)
|
|
if err != nil || !info.Mode().IsRegular() {
|
|
return "", false
|
|
}
|
|
resolvedRoot, err := filepath.EvalSymlinks(absRoot)
|
|
if err != nil {
|
|
return "", false
|
|
}
|
|
resolvedPath, err := filepath.EvalSymlinks(absPath)
|
|
if err != nil {
|
|
return "", false
|
|
}
|
|
rel, err := filepath.Rel(resolvedRoot, resolvedPath)
|
|
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
|
return "", false
|
|
}
|
|
resolvedInfo, err := os.Lstat(resolvedPath)
|
|
if err != nil || !resolvedInfo.Mode().IsRegular() {
|
|
return "", false
|
|
}
|
|
return filepath.Clean(resolvedPath), true
|
|
}
|
|
|
|
func hasUppercase(value string) bool {
|
|
for _, r := range value {
|
|
if r >= 'A' && r <= 'Z' {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|