The events table had no attempt column, so two attempts of the same task were indistinguishable: a retried task's stage events interleaved into one timeline and nothing could say which run produced a given result. - events gains an `attempt` column through the existing presence-gated migrateTasks loop rather than a version bump, so user_version stays 3 and an existing archie.db is migrated in place. A legacy-database test builds the pre-change schema and proves both that existing rows survive unattributed (attempt = 0) and that a new value round-trips. - Event.attempt = 11 and ReadTaskLogRequest.stage = 9 are field additions to existing messages: no new RPC, no new table, and no change to any canonical error message that crosses the state-store wire. - Every task event the workflow engine emits now carries the attempt. Producers that have no attempt yet (task_queued, binding_dispatch_failure) stay unattributed rather than guessing one. - Changed files are captured as a durable changes_captured event at commit and push time, reusing the existing three-dot diff primitive: paths from FilePatches(), counts from Stats(), base_sha recorded as the merge base actually diffed rather than the base tip, a 200-file cap with totals kept complete. Consumption goes through an unexported optional interface so workflow.Trees -- the yaegi capability surface for interpreted stage plugins -- is not widened. A capture is also taken once the pull request exists, so the capture can link it. - The per-attempt configuration is captured as a durable config_captured event carrying the non-secret config.TaskConfig, instead of a new table plus two new state-store RPCs: events are already this system's per-attempt provenance stream, so a table would have given one feature two provenance mechanisms. Redaction is proven against canary secrets. Measured payload: 539 bytes typical, 2333 bytes at 20 models. - Captures carry a schema marker and are refused unless their key set matches the declared payload shape, so drift surfaces as "recorded but could not be read" rather than as a capture of zeroes. - Three read endpoints serve the attempt rail, the changed-file captures and a raw debug view, mapping to UI-owned response types rather than dumping store types. - The task-log read gains a stage filter, bound to the stage logger for that stage's own lines; agent and tool output records no stage. A task-scoped grant may already insert events on its own task, so this provenance is exactly as trustworthy as the rest of that task's event stream. That limit is documented rather than papered over.
246 lines
7.9 KiB
Go
246 lines
7.9 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
)
|
|
|
|
const eventsSchema = `
|
|
CREATE TABLE IF NOT EXISTS events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
at TEXT NOT NULL,
|
|
kind TEXT NOT NULL,
|
|
task_id INTEGER NOT NULL DEFAULT 0,
|
|
repo TEXT NOT NULL DEFAULT '',
|
|
issue INTEGER NOT NULL DEFAULT 0,
|
|
workflow TEXT NOT NULL DEFAULT '',
|
|
stage TEXT NOT NULL DEFAULT '',
|
|
attempt INTEGER NOT NULL DEFAULT 0,
|
|
detail TEXT NOT NULL DEFAULT '',
|
|
data TEXT NOT NULL DEFAULT '{}'
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_events_task ON events(task_id, id);
|
|
CREATE INDEX IF NOT EXISTS idx_events_kind ON events(kind, id);
|
|
`
|
|
|
|
// InsertEvent appends an event to the log and returns its row id.
|
|
func (s *Store) InsertEvent(ctx context.Context, e events.Event) (int64, error) {
|
|
return insertEvent(ctx, s.db, e)
|
|
}
|
|
|
|
type eventExecer interface {
|
|
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
|
}
|
|
|
|
func insertEvent(ctx context.Context, execer eventExecer, e events.Event) (int64, error) {
|
|
if e.At.IsZero() {
|
|
e.At = time.Now().UTC()
|
|
}
|
|
data, err := json.Marshal(e.Data)
|
|
if err != nil {
|
|
data = fmt.Appendf(nil, `{"marshal_error":%q}`, err.Error())
|
|
}
|
|
res, err := execer.ExecContext(ctx, `
|
|
INSERT INTO events (at, kind, task_id, repo, issue, workflow, stage, attempt, detail, data)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
e.At.UTC().Format(time.RFC3339Nano), e.Kind, e.TaskID, e.Repo, e.Issue,
|
|
e.Workflow, e.Stage, e.Attempt, clip(e.Detail, 4000), string(data))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return res.LastInsertId()
|
|
}
|
|
|
|
func scanEvents(rows *sql.Rows) (out []events.Event, retErr error) {
|
|
defer func() {
|
|
retErr = errors.Join(retErr, rows.Close())
|
|
}()
|
|
for rows.Next() {
|
|
var e events.Event
|
|
var at, data string
|
|
if err := rows.Scan(&e.ID, &at, &e.Kind, &e.TaskID, &e.Repo, &e.Issue,
|
|
&e.Workflow, &e.Stage, &e.Attempt, &e.Detail, &data); err != nil {
|
|
return nil, err
|
|
}
|
|
e.At, _ = time.Parse(time.RFC3339Nano, at)
|
|
_ = json.Unmarshal([]byte(data), &e.Data)
|
|
out = append(out, e)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
const eventCols = "id, at, kind, task_id, repo, issue, workflow, stage, attempt, detail, data"
|
|
|
|
// EventsSince returns up to limit events with id > sinceID, oldest
|
|
// first -- SSE catch-up and the live feed.
|
|
func (s *Store) EventsSince(ctx context.Context, sinceID int64, limit int) ([]events.Event, error) {
|
|
rows, err := s.db.QueryContext(ctx,
|
|
`SELECT `+eventCols+` FROM events WHERE id > ? ORDER BY id LIMIT ?`, sinceID, limit)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return scanEvents(rows)
|
|
}
|
|
|
|
// TaskEvents returns a task's full timeline, oldest first.
|
|
func (s *Store) TaskEvents(ctx context.Context, taskID int64) ([]events.Event, error) {
|
|
rows, err := s.db.QueryContext(ctx,
|
|
`SELECT `+eventCols+` FROM events WHERE task_id = ? ORDER BY id`, taskID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return scanEvents(rows)
|
|
}
|
|
|
|
// Tasks returns all tasks, newest first (dashboard listing).
|
|
func (s *Store) Tasks(ctx context.Context, limit int) (tasks []workflow.Task, retErr error) {
|
|
rows, err := s.db.QueryContext(ctx, `
|
|
SELECT id, owner, repo, issue_number, title, status, workflow, stage,
|
|
pr_number, tokens_used, iterations, attempt, park_reason, retry_count,
|
|
created_at, updated_at, plan, source, identity, binding_id, binding_version
|
|
FROM tasks ORDER BY updated_at DESC LIMIT ?`, limit)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
retErr = errors.Join(retErr, rows.Close())
|
|
}()
|
|
for rows.Next() {
|
|
var t workflow.Task
|
|
// Plan and Source are read by the dashboard: Plan gates the
|
|
// "Decision required" panel on a waiting_human task, and Source
|
|
// decides whether an issue link is rendered at all -- a chat task's
|
|
// issue number is synthetic and links nowhere.
|
|
if err := rows.Scan(&t.ID, &t.Owner, &t.Repo, &t.IssueNumber, &t.Title,
|
|
&t.Status, &t.Workflow, &t.Stage, &t.PRNumber, &t.TokensUsed,
|
|
&t.Iterations, &t.Attempt, &t.ParkReason, &t.RetryCount,
|
|
sqliteTime{&t.CreatedAt}, sqliteTime{&t.UpdatedAt},
|
|
&t.Plan, &t.Source, &t.Identity, &t.BindingID, &t.BindingVersion); err != nil {
|
|
return nil, err
|
|
}
|
|
tasks = append(tasks, t)
|
|
}
|
|
return tasks, rows.Err()
|
|
}
|
|
|
|
// StatusCounts returns task counts by status.
|
|
func (s *Store) StatusCounts(ctx context.Context) (counts map[string]int, retErr error) {
|
|
rows, err := s.db.QueryContext(ctx, `SELECT status, COUNT(*) FROM tasks GROUP BY status`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
retErr = errors.Join(retErr, rows.Close())
|
|
}()
|
|
counts = map[string]int{}
|
|
for rows.Next() {
|
|
var st string
|
|
var n int
|
|
if err := rows.Scan(&st, &n); err != nil {
|
|
return nil, err
|
|
}
|
|
counts[st] = n
|
|
}
|
|
return counts, rows.Err()
|
|
}
|
|
|
|
// WorkflowStats aggregates outcomes and spend per workflow -- the
|
|
// core "is archie getting better per dollar" table.
|
|
func (s *Store) WorkflowStats(ctx context.Context) (stats []WorkflowStat, retErr error) {
|
|
rows, err := s.db.QueryContext(ctx, `
|
|
SELECT workflow, COUNT(*),
|
|
SUM(status='merged'), SUM(status='pr_open'), SUM(status='parked'),
|
|
CAST(AVG(tokens_used) AS INTEGER), AVG(iterations), SUM(tokens_used)
|
|
FROM tasks WHERE workflow != '' GROUP BY workflow ORDER BY COUNT(*) DESC`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
retErr = errors.Join(retErr, rows.Close())
|
|
}()
|
|
for rows.Next() {
|
|
var w WorkflowStat
|
|
if err := rows.Scan(&w.Workflow, &w.Runs, &w.Merged, &w.PROpen, &w.Parked,
|
|
&w.AvgTokens, &w.AvgSteps, &w.TotalToken); err != nil {
|
|
return nil, err
|
|
}
|
|
stats = append(stats, w)
|
|
}
|
|
return stats, rows.Err()
|
|
}
|
|
|
|
// StageStats aggregates stage_finish events -- where does time go, and
|
|
// which stages fail.
|
|
func (s *Store) StageStats(ctx context.Context) (stats []StageStat, retErr error) {
|
|
rows, err := s.db.QueryContext(ctx, `
|
|
SELECT workflow, stage, COUNT(*),
|
|
CAST(AVG(json_extract(data,'$.duration_ms')) AS INTEGER),
|
|
SUM(CASE WHEN json_extract(data,'$.error') IS NOT NULL THEN 1 ELSE 0 END)
|
|
FROM events WHERE kind = 'stage_finish'
|
|
GROUP BY workflow, stage ORDER BY workflow, stage`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
retErr = errors.Join(retErr, rows.Close())
|
|
}()
|
|
for rows.Next() {
|
|
var st StageStat
|
|
var avg sql.NullInt64
|
|
if err := rows.Scan(&st.Workflow, &st.Stage, &st.Runs, &avg, &st.Errors); err != nil {
|
|
return nil, err
|
|
}
|
|
st.AvgMs = int(avg.Int64)
|
|
stats = append(stats, st)
|
|
}
|
|
return stats, rows.Err()
|
|
}
|
|
|
|
// TokensByDay sums task token spend per UTC day. Agent-finish events are
|
|
// the per-run accounting source; tasks.tokens_used is the durable fallback for
|
|
// older or remote runs whose events were not persisted.
|
|
func (s *Store) TokensByDay(ctx context.Context, days int) (tokens []DayTokens, retErr error) {
|
|
tokens = make([]DayTokens, 0)
|
|
rows, err := s.db.QueryContext(ctx, `
|
|
WITH event_totals AS (
|
|
SELECT task_id, substr(at,1,10) AS day,
|
|
SUM(CASE WHEN json_type(data,'$.tokens') IN ('integer','real')
|
|
THEN CAST(json_extract(data,'$.tokens') AS INTEGER) ELSE 0 END) AS tokens,
|
|
SUM(CASE WHEN json_type(data,'$.tokens') IN ('integer','real') THEN 1 ELSE 0 END) AS valid_tokens
|
|
FROM events WHERE kind = 'agent_finish'
|
|
GROUP BY task_id, day
|
|
), usage AS (
|
|
SELECT day, tokens FROM event_totals WHERE valid_tokens > 0
|
|
UNION ALL
|
|
SELECT substr(t.updated_at,1,10), t.tokens_used
|
|
FROM tasks t
|
|
WHERE t.tokens_used != 0 AND NOT EXISTS (
|
|
SELECT 1 FROM event_totals e
|
|
WHERE e.task_id = t.id AND e.valid_tokens > 0
|
|
)
|
|
)
|
|
SELECT day, CAST(SUM(tokens) AS INTEGER)
|
|
FROM usage GROUP BY day ORDER BY day DESC LIMIT ?`, days)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
retErr = errors.Join(retErr, rows.Close())
|
|
}()
|
|
for rows.Next() {
|
|
var d DayTokens
|
|
if err := rows.Scan(&d.Day, &d.Tokens); err != nil {
|
|
return nil, err
|
|
}
|
|
tokens = append(tokens, d)
|
|
}
|
|
return tokens, rows.Err()
|
|
}
|