fix(modelloop): log each model call and tool call as it finishes (#1850)
This commit is contained in:
2 files changed
+70
-4
No files matched your search
@@ -77,11 +77,12 @@ func (r *LoopRunner) Run(ctx context.Context, workspace string, req agentrun.Req
|
||||
return agentrun.Result{}, fmt.Errorf("build central tool set: %w", err)
|
||||
}
|
||||
captureTools := captureToolSet(req.CaptureTools, captures)
|
||||
var modelHooks []core.ModelHook
|
||||
var toolHooks []core.ToolHook
|
||||
log := r.logger(req)
|
||||
progress := newProgressLog(log)
|
||||
modelHooks, toolHooks := []core.ModelHook{progress}, []core.ToolHook{progress}
|
||||
if report.Transcript != nil {
|
||||
recorder := newTranscript(report.Transcript)
|
||||
modelHooks, toolHooks = []core.ModelHook{recorder}, []core.ToolHook{recorder}
|
||||
modelHooks, toolHooks = append(modelHooks, recorder), append(toolHooks, recorder)
|
||||
}
|
||||
res, err := r.run(ctx, agentloop.Config{
|
||||
Runtime: r.runtime,
|
||||
@@ -113,7 +114,7 @@ func (r *LoopRunner) Run(ctx context.Context, workspace string, req agentrun.Req
|
||||
),
|
||||
ModelHooks: modelHooks,
|
||||
ToolHooks: toolHooks,
|
||||
Logger: r.logger(req),
|
||||
Logger: log,
|
||||
})
|
||||
result := resultFromRun(req, res, notes.appended, captures)
|
||||
if cause := context.Cause(ctx); cause != nil {
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package modelloop
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/ai-sdk/chat"
|
||||
"github.com/samcharles93/ai-sdk/core"
|
||||
)
|
||||
|
||||
// progressLog writes one task-log line per model call and tool call as it
|
||||
// finishes. agentloop logs its per-step lines only after the run returns, so
|
||||
// without this the operator sees nothing for the length of the stage.
|
||||
type progressLog struct {
|
||||
log *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
started map[string]time.Time
|
||||
}
|
||||
|
||||
func newProgressLog(log *slog.Logger) *progressLog {
|
||||
return &progressLog{log: log, started: map[string]time.Time{}}
|
||||
}
|
||||
|
||||
func (p *progressLog) ModelCallStarted(context.Context, chat.Request) {}
|
||||
|
||||
func (p *progressLog) ModelCallFinished(ctx context.Context, _ chat.Request, resp *chat.Response, err error, latency time.Duration, usage *chat.Usage) {
|
||||
attrs := []any{"latency_ms", latency.Milliseconds()}
|
||||
if usage != nil {
|
||||
attrs = append(attrs, "prompt_tokens", usage.PromptTokens, "completion_tokens", usage.CompletionTokens)
|
||||
}
|
||||
if err != nil {
|
||||
p.log.WarnContext(ctx, "agent model call failed", append(attrs, "err", err)...)
|
||||
return
|
||||
}
|
||||
if resp != nil {
|
||||
attrs = append(attrs, "tool_calls", len(resp.ToolCalls))
|
||||
}
|
||||
p.log.InfoContext(ctx, "agent model call", attrs...)
|
||||
}
|
||||
|
||||
func (p *progressLog) BeforeToolExecute(_ context.Context, call *core.ToolCall) (*core.Skip, error) {
|
||||
p.mu.Lock()
|
||||
p.started[call.ToolCallID] = time.Now()
|
||||
p.mu.Unlock()
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *progressLog) AfterToolExecute(ctx context.Context, call core.ToolCall, result *core.ToolResult) {
|
||||
p.mu.Lock()
|
||||
started, ok := p.started[call.ToolCallID]
|
||||
delete(p.started, call.ToolCallID)
|
||||
p.mu.Unlock()
|
||||
attrs := []any{"tool", call.ToolName}
|
||||
if ok {
|
||||
attrs = append(attrs, "latency_ms", time.Since(started).Milliseconds())
|
||||
}
|
||||
if result.Error != "" {
|
||||
p.log.WarnContext(ctx, "agent tool call failed", append(attrs, "err", clip(result.Error, nil))...)
|
||||
return
|
||||
}
|
||||
p.log.InfoContext(ctx, "agent tool call", attrs...)
|
||||
}
|
||||
Reference in new issue
Block a user