fix(modelloop): log each model call and tool call as it finishes (#1850)
This commit is contained in:
2 files changed
+70
-4
No files matched your search
@@ -77,11 +77,12 @@ func (r *LoopRunner) Run(ctx context.Context, workspace string, req agentrun.Req
|
|||||||
return agentrun.Result{}, fmt.Errorf("build central tool set: %w", err)
|
return agentrun.Result{}, fmt.Errorf("build central tool set: %w", err)
|
||||||
}
|
}
|
||||||
captureTools := captureToolSet(req.CaptureTools, captures)
|
captureTools := captureToolSet(req.CaptureTools, captures)
|
||||||
var modelHooks []core.ModelHook
|
log := r.logger(req)
|
||||||
var toolHooks []core.ToolHook
|
progress := newProgressLog(log)
|
||||||
|
modelHooks, toolHooks := []core.ModelHook{progress}, []core.ToolHook{progress}
|
||||||
if report.Transcript != nil {
|
if report.Transcript != nil {
|
||||||
recorder := newTranscript(report.Transcript)
|
recorder := newTranscript(report.Transcript)
|
||||||
modelHooks, toolHooks = []core.ModelHook{recorder}, []core.ToolHook{recorder}
|
modelHooks, toolHooks = append(modelHooks, recorder), append(toolHooks, recorder)
|
||||||
}
|
}
|
||||||
res, err := r.run(ctx, agentloop.Config{
|
res, err := r.run(ctx, agentloop.Config{
|
||||||
Runtime: r.runtime,
|
Runtime: r.runtime,
|
||||||
@@ -113,7 +114,7 @@ func (r *LoopRunner) Run(ctx context.Context, workspace string, req agentrun.Req
|
|||||||
),
|
),
|
||||||
ModelHooks: modelHooks,
|
ModelHooks: modelHooks,
|
||||||
ToolHooks: toolHooks,
|
ToolHooks: toolHooks,
|
||||||
Logger: r.logger(req),
|
Logger: log,
|
||||||
})
|
})
|
||||||
result := resultFromRun(req, res, notes.appended, captures)
|
result := resultFromRun(req, res, notes.appended, captures)
|
||||||
if cause := context.Cause(ctx); cause != nil {
|
if cause := context.Cause(ctx); cause != nil {
|
||||||
|
|||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package modelloop
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/samcharles93/ai-sdk/chat"
|
||||||
|
"github.com/samcharles93/ai-sdk/core"
|
||||||
|
)
|
||||||
|
|
||||||
|
// progressLog writes one task-log line per model call and tool call as it
|
||||||
|
// finishes. agentloop logs its per-step lines only after the run returns, so
|
||||||
|
// without this the operator sees nothing for the length of the stage.
|
||||||
|
type progressLog struct {
|
||||||
|
log *slog.Logger
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
started map[string]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func newProgressLog(log *slog.Logger) *progressLog {
|
||||||
|
return &progressLog{log: log, started: map[string]time.Time{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *progressLog) ModelCallStarted(context.Context, chat.Request) {}
|
||||||
|
|
||||||
|
func (p *progressLog) ModelCallFinished(ctx context.Context, _ chat.Request, resp *chat.Response, err error, latency time.Duration, usage *chat.Usage) {
|
||||||
|
attrs := []any{"latency_ms", latency.Milliseconds()}
|
||||||
|
if usage != nil {
|
||||||
|
attrs = append(attrs, "prompt_tokens", usage.PromptTokens, "completion_tokens", usage.CompletionTokens)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
p.log.WarnContext(ctx, "agent model call failed", append(attrs, "err", err)...)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if resp != nil {
|
||||||
|
attrs = append(attrs, "tool_calls", len(resp.ToolCalls))
|
||||||
|
}
|
||||||
|
p.log.InfoContext(ctx, "agent model call", attrs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *progressLog) BeforeToolExecute(_ context.Context, call *core.ToolCall) (*core.Skip, error) {
|
||||||
|
p.mu.Lock()
|
||||||
|
p.started[call.ToolCallID] = time.Now()
|
||||||
|
p.mu.Unlock()
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *progressLog) AfterToolExecute(ctx context.Context, call core.ToolCall, result *core.ToolResult) {
|
||||||
|
p.mu.Lock()
|
||||||
|
started, ok := p.started[call.ToolCallID]
|
||||||
|
delete(p.started, call.ToolCallID)
|
||||||
|
p.mu.Unlock()
|
||||||
|
attrs := []any{"tool", call.ToolName}
|
||||||
|
if ok {
|
||||||
|
attrs = append(attrs, "latency_ms", time.Since(started).Milliseconds())
|
||||||
|
}
|
||||||
|
if result.Error != "" {
|
||||||
|
p.log.WarnContext(ctx, "agent tool call failed", append(attrs, "err", clip(result.Error, nil))...)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.log.InfoContext(ctx, "agent tool call", attrs...)
|
||||||
|
}
|
||||||
Reference in new issue
Block a user