fix(messaging): transcribe voice notes on the model-owning side of the boundary
The Messaging Service built the voice-transcription client in-process from [models].transcription and [providers.*], which the messaging boundary PRD reserves to the process that owns the model runtime and its provider credentials. The deletion gate only measured linked packages, so the PRD violation shipped green: config reads are not imports. Move the capability across the boundary: - archiemessaging no longer reads [models] or [providers], builds no transcription client, and drops the ResolvedConfig.Transcriber field. - telegram classifies a voice note as messaging.MediaTypeVoice and stops transcribing; it carries the attachment bytes and its note to the turn, which is all a frontend without the credential can do. - archied builds the transcriber beside the chat runtime from its own model-role configuration (setupTranscriber) and gives it to every gateway TurnRunner. - the turn runner transcribes a speech attachment before recording the inbound message, so stored history and the model context carry the transcript; a nil/failed/empty transcription keeps the frontend note. - messaging.TranscribedMessageText renders the transcript into the text a frontend built from its media note, preserving the sender's caption. Red first: TestTurnRunnerTranscribesSpeechOnTheModelOwningSide failed at 'Transcribe calls = 0, want 1' with the transcription call disabled.
This commit is contained in:
1 parent
8eab196b8d
commit
52bbb74bf2
15 files changed
+430
-215
No files matched your search
@@ -41,6 +41,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/domain/health"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
domainmemory "github.com/samcharles93/archie-core/internal/domain/memory"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
"github.com/samcharles93/archie-core/internal/domain/scheduling"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
@@ -61,6 +62,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/taskactions"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/toolbuilder"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/transcription"
|
||||
"github.com/samcharles93/archie-core/internal/logging"
|
||||
"github.com/samcharles93/archie-core/internal/plugin"
|
||||
"github.com/samcharles93/archie-core/internal/plugin/pluginextract"
|
||||
@@ -254,6 +256,15 @@ type boot struct {
|
||||
// unrelated call.
|
||||
embeddings domainembedding.Client
|
||||
|
||||
// transcriber is the optional voice-transcription capability. It is built
|
||||
// beside the chat runtime from this process's own [models]/[providers],
|
||||
// which is the model-owning side of the Messaging boundary: a channel
|
||||
// frontend carries a voice note's bytes across the inbound wire, and this
|
||||
// process turns them into text before the turn is recorded. Nil when no
|
||||
// role is configured or the provider credential did not resolve; a turn
|
||||
// then keeps the frontend's media note rather than failing.
|
||||
transcriber messaging.Transcriber
|
||||
|
||||
toolReg *tools.Registry
|
||||
chatModels *chatModelManager
|
||||
personas *gateway.PersonaRegistry
|
||||
@@ -872,6 +883,26 @@ func (b *boot) setupEmbeddings(cfg config.Config, log *slog.Logger) {
|
||||
}
|
||||
}
|
||||
|
||||
// setupTranscriber builds the optional voice-transcription capability. It is
|
||||
// the model-owning side of the Messaging Service boundary: the channel
|
||||
// frontend carries a voice note's audio bytes across the inbound wire but
|
||||
// holds no provider credential, so this process turns them into text before
|
||||
// the turn's message is recorded. Like setupEmbeddings, a client is wired only
|
||||
// when models["transcription"] names a provider/model and that provider's
|
||||
// credential resolves; a configured-but-unusable role is logged so the
|
||||
// degradation is visible, while a role never configured is silent.
|
||||
func (b *boot) setupTranscriber(cfg config.Config, log *slog.Logger) {
|
||||
client, ok := transcription.New(cfg.Models, cfg.Providers, transcription.Options{
|
||||
ResolveSecret: b.secrets.Resolve,
|
||||
})
|
||||
if ok {
|
||||
b.transcriber = client
|
||||
log.Info("voice transcription enabled", "role", transcription.Role)
|
||||
} else if cfg.Models[transcription.Role] != "" {
|
||||
log.Warn("voice transcription configured but unavailable; capability disabled", "role", transcription.Role)
|
||||
}
|
||||
}
|
||||
|
||||
// setupLLMAndChat wires the runtime, tool registry, model management,
|
||||
// personas and the dashboard's chat service.
|
||||
func (b *boot) setupLLMAndChat(ctx context.Context) error {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/daemon"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/gateway"
|
||||
"github.com/samcharles93/archie-core/internal/installtype"
|
||||
@@ -63,6 +64,12 @@ type chatSetup struct {
|
||||
// process makes, which the /status health source reads back. Nil disables
|
||||
// recording (the recorder's methods tolerate that rather than panicking).
|
||||
ProviderOutcomes *providerOutcomeRecorder
|
||||
// Transcriber is the voice-transcription capability built by
|
||||
// setupChatRuntime from this process's own [models]/[providers]. Every turn
|
||||
// runner carries it so a speech attachment is transcribed before its
|
||||
// message is recorded. Nil disables the capability (no role configured, or
|
||||
// the provider credential did not resolve), leaving the frontend's note.
|
||||
Transcriber messaging.Transcriber
|
||||
}
|
||||
|
||||
// daemonRunningVersions reports the component versions this process can
|
||||
@@ -190,6 +197,7 @@ func newChatTurnRunner(
|
||||
MemoryEngine: s.MemoryEngine,
|
||||
MemoryWriter: s.MemoryWriter,
|
||||
UserIdentity: userIdentityResolver(),
|
||||
Transcriber: s.Transcriber,
|
||||
Log: s.Log,
|
||||
})
|
||||
if err := runner.Recover(ctx); err != nil && s.Log != nil {
|
||||
|
||||
@@ -25,6 +25,11 @@ func (b *boot) setupChatRuntime(ctx context.Context, cfg config.Config) error {
|
||||
// non-command message processing.
|
||||
providers := executionProviders(cfg)
|
||||
b.setLLM(agentexec.NewRuntime(providers))
|
||||
// Transcription is a model-role capability, like the chat models it sits
|
||||
// beside: it is built here, on the model-owning side, from this process's
|
||||
// own [models]/[providers]. The Messaging Service carries a voice note's
|
||||
// bytes to the Gateway and never holds the credential.
|
||||
b.setupTranscriber(cfg, b.log)
|
||||
b.toolReg = tools.NewRegistry()
|
||||
chatModels := newChatModelManager(cfg.Models, cfg.Chat.Models)
|
||||
catalog, catalogModels := b.catalogState()
|
||||
@@ -142,6 +147,9 @@ func (b *boot) setupGatewayChat(ctx context.Context, actor gateway.ChatTaskActor
|
||||
Bus: b.bus, Log: b.log,
|
||||
MemoryEngine: b.memoryStore(),
|
||||
MemoryWriter: b.memoryWriter(),
|
||||
// The transcription capability is built beside the chat runtime, from
|
||||
// the model-role configuration only this process reads.
|
||||
Transcriber: b.transcriber,
|
||||
// The Gateway executes the web chat's turns, so it is the process
|
||||
// that must record their outcomes for /status.
|
||||
ProviderOutcomes: b.providerOutcomes,
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/transcription"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
)
|
||||
|
||||
// TestSetupTranscriberBuildsOnTheModelOwningSide pins where the voice
|
||||
// capability is constructed: beside the chat runtime, from the process's own
|
||||
// [models].transcription and [providers.*]. The Messaging Service carries the
|
||||
// audio bytes across the inbound wire but no longer builds this client.
|
||||
func TestSetupTranscriberBuildsOnTheModelOwningSide(t *testing.T) {
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
|
||||
t.Run("configured role wires a transcriber", func(t *testing.T) {
|
||||
b := &boot{secrets: secret.NewRegistry()}
|
||||
b.setupTranscriber(config.Config{
|
||||
Models: map[string]string{transcription.Role: "local/whisper-large-v3"},
|
||||
Providers: map[string]config.Provider{
|
||||
"local": {Class: "openai-compatible", BaseURL: "http://127.0.0.1:8080/v1"},
|
||||
},
|
||||
}, log)
|
||||
if b.transcriber == nil {
|
||||
t.Fatal("transcriber = nil, want a client built from [models].transcription")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("absent role degrades to nil", func(t *testing.T) {
|
||||
b := &boot{secrets: secret.NewRegistry()}
|
||||
b.setupTranscriber(config.Config{}, log)
|
||||
if b.transcriber != nil {
|
||||
t.Fatalf("transcriber = %v, want nil when no role is configured", b.transcriber)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -82,7 +82,6 @@ func compose(ctx context.Context, d deps) (*Service, error) {
|
||||
}
|
||||
tg := telegram.New(d.Config.TelegramToken, d.Config.Telegram.AllowedUserIDs, d.Log)
|
||||
tg.Settings = d.Settings
|
||||
tg.Transcriber = d.Config.Transcriber
|
||||
configureTelegram(ctx, tg, d.Config, d.Chat, d.Log)
|
||||
if err := srv.add("telegram", tg, telegramValidateConfigMap(d.Config.Telegram)); err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -8,9 +8,7 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/transcription"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
)
|
||||
|
||||
@@ -32,11 +30,6 @@ type ResolvedConfig struct {
|
||||
BotUser string
|
||||
HealthURL string
|
||||
ShowToolCalls bool
|
||||
// Transcriber is the optional voice-transcription capability. Nil means it
|
||||
// is not configured (or its provider did not resolve), and a voice note
|
||||
// keeps its media note. Constructed from [models].transcription and the
|
||||
// matching [providers.*] entry.
|
||||
Transcriber messaging.Transcriber
|
||||
}
|
||||
|
||||
type projection struct {
|
||||
@@ -50,8 +43,6 @@ type projection struct {
|
||||
botUser string
|
||||
healthURL string
|
||||
showToolCalls bool
|
||||
models map[string]string
|
||||
providers map[string]config.Provider
|
||||
}
|
||||
|
||||
// Resolve loads the configuration, extracts the messaging projection, resolves
|
||||
@@ -101,7 +92,6 @@ func Resolve(o Options, log *slog.Logger) (ResolvedConfig, error) {
|
||||
BotUser: proj.botUser,
|
||||
HealthURL: proj.healthURL,
|
||||
ShowToolCalls: proj.showToolCalls,
|
||||
Transcriber: setupTranscriber(proj, secrets, log),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -131,8 +121,6 @@ func project(cfg config.Config) projection {
|
||||
botUser: cfg.BotUser,
|
||||
healthURL: cfg.Health.URL(),
|
||||
showToolCalls: cfg.Chat.ShowToolCalls,
|
||||
models: cfg.Models,
|
||||
providers: cfg.Providers,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,20 +165,3 @@ func resolveWebhookSecret(route config.WebhookRoute, secrets *secret.Registry) (
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// setupTranscriber builds the optional voice-transcription capability. A
|
||||
// client is wired only when [models].transcription names a provider/model and
|
||||
// that provider's credential resolves; a configured-but-unusable role is
|
||||
// logged so the degradation is visible, while a role that was never
|
||||
// configured is silent (mirrors archied's setupEmbeddings).
|
||||
func setupTranscriber(proj projection, secrets *secret.Registry, log *slog.Logger) messaging.Transcriber {
|
||||
client, ok := transcription.New(proj.models, proj.providers, transcription.Options{ResolveSecret: secrets.Resolve})
|
||||
if ok {
|
||||
log.Info("voice transcription enabled", "role", transcription.Role)
|
||||
return client
|
||||
}
|
||||
if proj.models[transcription.Role] != "" {
|
||||
log.Warn("voice transcription configured but unavailable; capability disabled", "role", transcription.Role)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -72,70 +72,6 @@ allowed_user_ids = [12345]
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBuildsTranscriberFromModelRole(t *testing.T) {
|
||||
write := func(t *testing.T, extra string) string {
|
||||
t.Helper()
|
||||
tmp := t.TempDir()
|
||||
path := filepath.Join(tmp, "config.toml")
|
||||
content := `
|
||||
bot_user = "testbot"
|
||||
|
||||
[services.gateway]
|
||||
target = "127.0.0.1:8999"
|
||||
` + extra
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
t.Run("configured role wires a transcriber", func(t *testing.T) {
|
||||
path := write(t, `
|
||||
[models]
|
||||
transcription = "local/whisper-large-v3"
|
||||
|
||||
[providers.local]
|
||||
class = "openai-compatible"
|
||||
base_url = "http://127.0.0.1:8080/v1"
|
||||
`)
|
||||
resolved, err := Resolve(Options{Config: path}, slog.Default())
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
}
|
||||
if resolved.Transcriber == nil {
|
||||
t.Error("Transcriber = nil, want a client built from [models].transcription")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("absent role degrades to nil", func(t *testing.T) {
|
||||
resolved, err := Resolve(Options{Config: write(t, "")}, slog.Default())
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
}
|
||||
if resolved.Transcriber != nil {
|
||||
t.Errorf("Transcriber = %v, want nil when no role is configured", resolved.Transcriber)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("configured but unusable role degrades to nil", func(t *testing.T) {
|
||||
path := write(t, `
|
||||
[models]
|
||||
transcription = "openai/whisper-1"
|
||||
|
||||
[providers.openai]
|
||||
class = "openai"
|
||||
api_key_env = "TRANSCRIPTION_TEST_UNSET_KEY"
|
||||
`)
|
||||
resolved, err := Resolve(Options{Config: path}, slog.Default())
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
}
|
||||
if resolved.Transcriber != nil {
|
||||
t.Errorf("Transcriber = %v, want nil when the credential is missing", resolved.Transcriber)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestServiceStartAndStop(t *testing.T) {
|
||||
d := deps{
|
||||
Config: ResolvedConfig{
|
||||
|
||||
@@ -21,9 +21,6 @@ type inboundMedia struct {
|
||||
// note is a bracketed, self-describing placeholder such as "[photo]"
|
||||
// or "[document: report.pdf]".
|
||||
note string
|
||||
// transcribe marks audio a configured Transcriber should turn into text
|
||||
// (a Telegram voice note). False leaves the note as-is.
|
||||
transcribe bool
|
||||
}
|
||||
|
||||
// extractInboundMedia maps a Telegram message onto the attachment it
|
||||
@@ -73,12 +70,12 @@ func extractInboundMedia(msg *models.Message) (inboundMedia, bool) {
|
||||
|
||||
case msg.Voice != nil:
|
||||
return inboundMedia{attachment: messaging.MediaAttachment{
|
||||
Type: "audio",
|
||||
Type: messaging.MediaTypeVoice,
|
||||
FileID: msg.Voice.FileID,
|
||||
MIMEType: msg.Voice.MimeType,
|
||||
FileSize: nilIfZero(msg.Voice.FileSize),
|
||||
Duration: &msg.Voice.Duration,
|
||||
}, note: "[voice message]", transcribe: true}, true
|
||||
}, note: "[voice message]"}, true
|
||||
}
|
||||
return inboundMedia{}, false
|
||||
}
|
||||
|
||||
@@ -158,7 +158,7 @@ func TestExtractInboundMediaMapsTelegramKinds(t *testing.T) {
|
||||
msg: &models.Message{
|
||||
Voice: &models.Voice{FileID: "voice1", MimeType: "audio/ogg", Duration: 8},
|
||||
},
|
||||
wantType: "audio",
|
||||
wantType: messaging.MediaTypeVoice,
|
||||
wantFileID: "voice1",
|
||||
wantMIME: "audio/ogg",
|
||||
wantNote: "[voice message]",
|
||||
@@ -333,24 +333,6 @@ func TestDefaultHandlerSilentForNonContentMessages(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// fakeTranscriber records the audio it was handed and returns a fixed
|
||||
// transcript, so a test can assert both that transcription ran on the
|
||||
// downloaded bytes and what the turn then carried.
|
||||
type fakeTranscriber struct {
|
||||
text string
|
||||
err error
|
||||
calls int
|
||||
audio []byte
|
||||
}
|
||||
|
||||
var _ messaging.Transcriber = (*fakeTranscriber)(nil)
|
||||
|
||||
func (f *fakeTranscriber) Transcribe(_ context.Context, audio []byte) (string, error) {
|
||||
f.calls++
|
||||
f.audio = append([]byte(nil), audio...)
|
||||
return f.text, f.err
|
||||
}
|
||||
|
||||
// voiceUpdate is a Telegram voice note from the authorized sender.
|
||||
func voiceUpdate(fileID string) *models.Message {
|
||||
return &models.Message{
|
||||
@@ -362,11 +344,12 @@ func voiceUpdate(fileID string) *models.Message {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultHandlerTranscribesVoiceNoteIntoText drives the behaviour this
|
||||
// feature exists for: a configured Transcriber turns the downloaded voice
|
||||
// note into the turn's text, behind a provenance marker so the agent knows
|
||||
// it is reading a machine transcript.
|
||||
func TestDefaultHandlerTranscribesVoiceNoteIntoText(t *testing.T) {
|
||||
// TestDefaultHandlerMarksVoiceNoteAsSpeech drives the frontend half of the
|
||||
// boundary: the messaging process downloads the note, classifies it as speech
|
||||
// (messaging.MediaTypeVoice), and carries the bytes and the note to the turn.
|
||||
// Turning those bytes into text is the model-owning process's job -- this
|
||||
// process holds no provider credential -- so no transcription happens here.
|
||||
func TestDefaultHandlerMarksVoiceNoteAsSpeech(t *testing.T) {
|
||||
voiceBytes := []byte("ogg-voice-bytes")
|
||||
b, _ := newRecordedAPI(t, func(w http.ResponseWriter, r *http.Request) bool {
|
||||
if strings.HasPrefix(r.URL.Path, "/file/bot1:test/") {
|
||||
@@ -386,72 +369,30 @@ func TestDefaultHandlerTranscribesVoiceNoteIntoText(t *testing.T) {
|
||||
return ch, nil
|
||||
},
|
||||
}
|
||||
transcriber := &fakeTranscriber{text: "turn the lights on"}
|
||||
g := testMediaGateway(b)
|
||||
g.Transcriber = transcriber
|
||||
|
||||
g.defaultHandler(client)(context.Background(), b, &models.Update{Message: voiceUpdate("voice1")})
|
||||
|
||||
select {
|
||||
case in := <-inboundCh:
|
||||
want := "[voice transcription]\nturn the lights on"
|
||||
if in.Message.Text != want {
|
||||
t.Errorf("Text = %q, want %q", in.Message.Text, want)
|
||||
if len(in.Media) != 1 || in.Media[0].Type != messaging.MediaTypeVoice {
|
||||
t.Fatalf("inbound media = %#v, want the voice attachment marked as speech", in.Media)
|
||||
}
|
||||
if len(in.Media) != 1 || in.Media[0].Type != "audio" {
|
||||
t.Fatalf("inbound media = %#v, want the voice attachment", in.Media)
|
||||
if string(in.Media[0].Data) != string(voiceBytes) {
|
||||
t.Errorf("inbound media bytes = %q, want the downloaded voice bytes", in.Media[0].Data)
|
||||
}
|
||||
if transcriber.calls != 1 {
|
||||
t.Fatalf("Transcribe calls = %d, want 1", transcriber.calls)
|
||||
}
|
||||
if string(transcriber.audio) != string(voiceBytes) {
|
||||
t.Errorf("Transcribe received %q, want the downloaded voice bytes", transcriber.audio)
|
||||
if in.Message.Text != "[voice message]" {
|
||||
t.Errorf("Text = %q, want the media note preserved for the model-owning side", in.Message.Text)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("voice note never reached the chat turn")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultHandlerKeepsVoiceNoteWhenTranscriptionUnavailable pins the
|
||||
// degradation the config contract requires: with no Transcriber the voice
|
||||
// note still becomes a turn, carrying today's "[voice message]" note rather
|
||||
// than failing or going silent.
|
||||
func TestDefaultHandlerKeepsVoiceNoteWhenTranscriptionUnavailable(t *testing.T) {
|
||||
b, _ := newRecordedAPI(t, func(w http.ResponseWriter, r *http.Request) bool {
|
||||
if strings.HasPrefix(r.URL.Path, "/file/bot1:test/") {
|
||||
_, _ = w.Write([]byte("ogg-voice-bytes"))
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
|
||||
inboundCh := make(chan messaging.Inbound, 1)
|
||||
client := &fakeChatContract{
|
||||
streamFunc: func(_ context.Context, in messaging.Inbound) (<-chan messaging.ChatEvent, error) {
|
||||
inboundCh <- in
|
||||
ch := make(chan messaging.ChatEvent, 1)
|
||||
ch <- messaging.ChatEvent{Kind: "done", Text: "done", SessionID: "s1"}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
},
|
||||
}
|
||||
g := testMediaGateway(b) // Transcriber left nil on purpose
|
||||
|
||||
g.defaultHandler(client)(context.Background(), b, &models.Update{Message: voiceUpdate("voice1")})
|
||||
|
||||
select {
|
||||
case in := <-inboundCh:
|
||||
if in.Message.Text != "[voice message]" {
|
||||
t.Errorf("Text = %q, want the media note preserved", in.Message.Text)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("voice note was dropped when transcription was unavailable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultHandlerDoesNotTranscribePlainAudio keeps the flag scoped to
|
||||
// voice notes: a forwarded music file must not spend a transcription call.
|
||||
func TestDefaultHandlerDoesNotTranscribePlainAudio(t *testing.T) {
|
||||
// TestDefaultHandlerMarksPlainAudioAsAudio keeps the speech distinction: a
|
||||
// forwarded music file is ordinary audio, so the model-owning side knows not
|
||||
// to spend a transcription call on it.
|
||||
func TestDefaultHandlerMarksPlainAudioAsAudio(t *testing.T) {
|
||||
b, _ := newRecordedAPI(t, func(w http.ResponseWriter, r *http.Request) bool {
|
||||
if strings.HasPrefix(r.URL.Path, "/file/bot1:test/") {
|
||||
_, _ = w.Write([]byte("mp3-bytes"))
|
||||
@@ -470,9 +411,7 @@ func TestDefaultHandlerDoesNotTranscribePlainAudio(t *testing.T) {
|
||||
return ch, nil
|
||||
},
|
||||
}
|
||||
transcriber := &fakeTranscriber{text: "should not be used"}
|
||||
g := testMediaGateway(b)
|
||||
g.Transcriber = transcriber
|
||||
|
||||
g.defaultHandler(client)(context.Background(), b, &models.Update{Message: &models.Message{
|
||||
ID: 12,
|
||||
@@ -484,12 +423,12 @@ func TestDefaultHandlerDoesNotTranscribePlainAudio(t *testing.T) {
|
||||
|
||||
select {
|
||||
case in := <-inboundCh:
|
||||
if len(in.Media) != 1 || in.Media[0].Type != messaging.MediaTypeAudio {
|
||||
t.Fatalf("inbound media = %#v, want the plain audio attachment", in.Media)
|
||||
}
|
||||
if in.Message.Text != "[audio]" {
|
||||
t.Errorf("Text = %q, want the plain audio note", in.Message.Text)
|
||||
}
|
||||
if transcriber.calls != 0 {
|
||||
t.Errorf("Transcribe calls = %d, want 0 for non-voice audio", transcriber.calls)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("audio update never reached the chat turn")
|
||||
}
|
||||
|
||||
@@ -50,12 +50,6 @@ type Gateway struct {
|
||||
Updates UpdateService
|
||||
// Settings executes control-plane commands in this authenticated adapter.
|
||||
Settings *messaging.SettingsCommand
|
||||
// Transcriber turns voice notes into text before their turn is dispatched.
|
||||
// Nil means the capability is not configured (or its provider did not
|
||||
// resolve): a voice note then keeps its "[voice message]" note, exactly
|
||||
// the behaviour before transcription existed. The composition root owns
|
||||
// construction, so this adapter never holds a provider credential itself.
|
||||
Transcriber messaging.Transcriber
|
||||
// ResolveActor maps a channel-native sender to an authenticated Archie
|
||||
// identity. Nil deliberately permits reads but makes writes fail closed.
|
||||
ResolveActor func(int64) (string, bool)
|
||||
@@ -655,7 +649,6 @@ func (g *Gateway) submitTurn(ctx context.Context, b *bot.Bot, msg *models.Messag
|
||||
if !isTurn {
|
||||
return
|
||||
}
|
||||
transcribeVoice := hasMedia && media.transcribe
|
||||
|
||||
// The lane key must be the session, so that /stop -- which resolves
|
||||
// the same key -- reaches the turn the sender is actually watching.
|
||||
@@ -665,7 +658,6 @@ func (g *Gateway) submitTurn(ctx context.Context, b *bot.Bot, msg *models.Messag
|
||||
if !g.fetchTurnMedia(turnCtx, b, &gm, chatID, threadID) {
|
||||
return
|
||||
}
|
||||
g.transcribeVoiceNote(turnCtx, &gm, msg.Caption, transcribeVoice)
|
||||
|
||||
// If the turn invokes a tool that requires human approval,
|
||||
// the dispatch layer blocks on this approver. Nil is fine
|
||||
@@ -776,27 +768,6 @@ func (g *Gateway) fetchTurnMedia(ctx context.Context, b *bot.Bot, gm *messaging.
|
||||
return true
|
||||
}
|
||||
|
||||
// transcribeVoiceNote replaces a voice note's stored placeholder with the
|
||||
// transcript, so the agent reads what was said rather than "[voice message]".
|
||||
// It degrades: a nil Transcriber, an error, or an empty transcript leaves the
|
||||
// placeholder in place -- a turn must never fail because transcription was
|
||||
// unavailable.
|
||||
func (g *Gateway) transcribeVoiceNote(ctx context.Context, gm *messaging.Inbound, caption string, wanted bool) {
|
||||
if !wanted || g.Transcriber == nil || len(gm.Media) == 0 {
|
||||
return
|
||||
}
|
||||
transcript, err := g.Transcriber.Transcribe(ctx, gm.Media[0].Data)
|
||||
if err != nil {
|
||||
g.log.Warn("voice transcription failed; keeping the media note", "error", err)
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(transcript) == "" {
|
||||
g.log.Info("voice transcription produced no text; keeping the media note")
|
||||
return
|
||||
}
|
||||
gm.Message.Text = messaging.TranscribedText(transcript, caption)
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────
|
||||
|
||||
// conversationID addresses the chat a Telegram message arrived in: the
|
||||
|
||||
@@ -9,6 +9,21 @@ package messaging
|
||||
// attachment is accepted rather than rejected as an oversized message.
|
||||
const MaxInboundAttachmentBytes = 20 << 20
|
||||
|
||||
// Type values. MediaTypeVoice is speech the sender recorded -- a Telegram
|
||||
// voice note -- as distinct from MediaTypeAudio, a music or generic audio
|
||||
// file. The distinction is what tells the model-owning turn runner to
|
||||
// transcribe a speech attachment before the turn: a voice note's bytes reach
|
||||
// the Gateway, which owns the provider credential, but the Gateway must not
|
||||
// spend a transcription call on a forwarded song. The remaining values are
|
||||
// what they say.
|
||||
const (
|
||||
MediaTypeImage = "image"
|
||||
MediaTypeVideo = "video"
|
||||
MediaTypeAudio = "audio"
|
||||
MediaTypeDocument = "document"
|
||||
MediaTypeVoice = "voice"
|
||||
)
|
||||
|
||||
// MediaAttachment describes a file attached to a message. Platform-agnostic;
|
||||
// platform-specific fields live in Raw.
|
||||
//
|
||||
@@ -16,7 +31,7 @@ const MaxInboundAttachmentBytes = 20 << 20
|
||||
// "unknown" across a JSON round-trip: a 0-byte file is valid and
|
||||
// different from a file whose size was never set.
|
||||
type MediaAttachment struct {
|
||||
// Type is "image", "video", "audio", or "document".
|
||||
// Type is "image", "video", "audio", "voice", or "document".
|
||||
Type string `json:"type"`
|
||||
|
||||
// FileID is the platform-assigned identifier for download APIs.
|
||||
|
||||
@@ -51,3 +51,33 @@ func TranscribedText(transcript, caption string) string {
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
// TranscribedMessageText renders the text a transcribed speech attachment
|
||||
// contributes to a message, preserving any caption the sender attached.
|
||||
//
|
||||
// messageText is the inbound text a channel frontend rendered for an
|
||||
// attachment: the media note alone, or the note followed by the sender's
|
||||
// caption on the next line. The note is a bracketed, self-describing
|
||||
// placeholder (see internal/channels/telegram's turnMessageText); it is
|
||||
// replaced by the provenance marker and transcript, and anything after it
|
||||
// survives as the caption. Text that carries no recognisable note is treated
|
||||
// as a caption in full, so nothing the sender wrote is dropped.
|
||||
//
|
||||
// The transcription happens on the model-owning side of the Messaging
|
||||
// boundary, which receives the audio bytes but not the caption as a separate
|
||||
// field; this is the function that turns the frontend's rendered text into
|
||||
// the transcript the turn is recorded with.
|
||||
func TranscribedMessageText(messageText, transcript string) string {
|
||||
note, caption, _ := strings.Cut(messageText, "\n")
|
||||
if !isMediaNote(note) {
|
||||
return TranscribedText(transcript, messageText)
|
||||
}
|
||||
return TranscribedText(transcript, caption)
|
||||
}
|
||||
|
||||
// isMediaNote reports whether s is a channel frontend's bracketed media
|
||||
// placeholder such as "[voice message]".
|
||||
func isMediaNote(s string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
return len(s) >= 2 && s[0] == '[' && s[len(s)-1] == ']'
|
||||
}
|
||||
@@ -2,6 +2,45 @@ package messaging
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestTranscribedMessageTextReplacesTheMediaNote pins the boundary handoff:
|
||||
// the frontend renders a voice note as its bracketed note (plus any caption),
|
||||
// and the model-owning side only has that text when it transcribes. The note
|
||||
// must be replaced by the transcript; the caption must survive.
|
||||
func TestTranscribedMessageTextReplacesTheMediaNote(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
messageText string
|
||||
transcript string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "note alone is replaced",
|
||||
messageText: "[voice message]",
|
||||
transcript: "hello there",
|
||||
want: "[voice transcription]\nhello there",
|
||||
},
|
||||
{
|
||||
name: "caption after the note survives",
|
||||
messageText: "[voice message]\nignore the background noise",
|
||||
transcript: "what time is it",
|
||||
want: "[voice transcription]\nwhat time is it\nignore the background noise",
|
||||
},
|
||||
{
|
||||
name: "text with no note is kept as the caption",
|
||||
messageText: "typed it by hand",
|
||||
transcript: "spoken words",
|
||||
want: "[voice transcription]\nspoken words\ntyped it by hand",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := TranscribedMessageText(tc.messageText, tc.transcript); got != tc.want {
|
||||
t.Errorf("TranscribedMessageText(%q, %q) = %q, want %q", tc.messageText, tc.transcript, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestTranscribedTextMarksProvenanceAndKeepsCaption pins the stored shape of
|
||||
// a transcribed voice note: history is text-only and the original bytes are
|
||||
// gone, so the marker is the only thing that tells a later reader the words
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -87,11 +88,20 @@ const defaultTurnPersistenceTimeout = 5 * time.Second
|
||||
// runner. The runner is channel-neutral apart from the channel name used in
|
||||
// prompts, session tools, and completion events.
|
||||
type TurnRunnerConfig struct {
|
||||
Router *Router
|
||||
Sessions SessionStore
|
||||
Models ModelManager
|
||||
Personas PersonaPromptSource
|
||||
Model TurnModel
|
||||
Router *Router
|
||||
Sessions SessionStore
|
||||
Models ModelManager
|
||||
Personas PersonaPromptSource
|
||||
Model TurnModel
|
||||
// Transcriber turns an inbound speech attachment (messaging.MediaTypeVoice)
|
||||
// into text before the turn's inbound message is recorded, so stored
|
||||
// history reads what was said rather than the frontend's note. The
|
||||
// composition root builds it from the process's own [models]/[providers],
|
||||
// which only the model-owning side of the Messaging boundary reads; a
|
||||
// channel frontend carries the attachment bytes across the inbound wire
|
||||
// but never holds the provider credential. Nil keeps the note (the
|
||||
// capability is optional and degrades).
|
||||
Transcriber messaging.Transcriber
|
||||
Ledger TurnLedger
|
||||
OwnerID string
|
||||
PersistenceTimeout time.Duration
|
||||
@@ -485,6 +495,13 @@ func (r *TurnRunner) Run(ctx context.Context, in Inbound, stream TurnStream) (st
|
||||
return replayed, err
|
||||
}
|
||||
|
||||
// A message already recorded by an earlier attempt (turn.InputMessageID
|
||||
// set) keeps the transcript that attempt persisted; transcribing again
|
||||
// would duplicate the call and could disagree with stored history.
|
||||
if turn.InputMessageID == "" {
|
||||
in.Message = r.transcribeSpeech(ctx, in)
|
||||
}
|
||||
|
||||
in.Message, history, err = r.recordInboundMessage(ctx, &turn, sessionID, in.Message, history)
|
||||
if err != nil {
|
||||
return "", r.failTurn(ctx, turn, err)
|
||||
@@ -497,6 +514,39 @@ func (r *TurnRunner) Run(ctx context.Context, in Inbound, stream TurnStream) (st
|
||||
return r.generateAndComplete(ctx, turn, sessionID, prep, stream)
|
||||
}
|
||||
|
||||
// transcribeSpeech replaces an inbound speech attachment's placeholder note
|
||||
// with the transcript before the turn's message is recorded, so the store --
|
||||
// and every later turn reading its history -- keeps the words rather than a
|
||||
// note whose audio is gone. It degrades: a nil Transcriber, a failed call, or
|
||||
// an empty transcript leaves the frontend's note in place. A turn must never
|
||||
// fail because transcription was unavailable.
|
||||
func (r *TurnRunner) transcribeSpeech(ctx context.Context, in Inbound) messaging.Message {
|
||||
msg := in.Message
|
||||
if r.Transcriber == nil {
|
||||
return msg
|
||||
}
|
||||
for _, att := range in.Media {
|
||||
if att.Type != messaging.MediaTypeVoice || len(att.Data) == 0 {
|
||||
continue
|
||||
}
|
||||
transcript, err := r.Transcriber.Transcribe(ctx, att.Data)
|
||||
if err != nil {
|
||||
if r.Log != nil {
|
||||
r.Log.Warn("voice transcription failed; keeping the media note", "error", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(transcript) == "" {
|
||||
if r.Log != nil {
|
||||
r.Log.Info("voice transcription produced no text; keeping the media note")
|
||||
}
|
||||
continue
|
||||
}
|
||||
msg.Text = messaging.TranscribedMessageText(msg.Text, transcript)
|
||||
}
|
||||
return msg
|
||||
}
|
||||
|
||||
// generateAndComplete runs the model, persists the assistant message and
|
||||
// completed turn, and publishes the turn-completed event.
|
||||
func (r *TurnRunner) generateAndComplete(
|
||||
|
||||
@@ -2,6 +2,7 @@ package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"reflect"
|
||||
"testing"
|
||||
@@ -54,3 +55,184 @@ func TestTurnRunnerGivesInboundMediaToTheModelRequest(t *testing.T) {
|
||||
t.Errorf("model request media = %#v, want %#v", prepared.lastRequest.Media, media)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeTranscriber records the audio it is handed and returns a fixed
|
||||
// transcript, so a test can prove the model-owning turn runner transcribed
|
||||
// the bytes a channel frontend carried across the boundary.
|
||||
type fakeTranscriber struct {
|
||||
text string
|
||||
err error
|
||||
calls int
|
||||
audio []byte
|
||||
}
|
||||
|
||||
func (f *fakeTranscriber) Transcribe(_ context.Context, audio []byte) (string, error) {
|
||||
f.calls++
|
||||
f.audio = append([]byte(nil), audio...)
|
||||
return f.text, f.err
|
||||
}
|
||||
|
||||
// TestTurnRunnerTranscribesSpeechOnTheModelOwningSide drives the behaviour the
|
||||
// boundary move exists for: a voice attachment's bytes reach the process that
|
||||
// owns the model and provider credentials, which turns them into the turn's
|
||||
// text before the inbound message is recorded. The stored history -- the only
|
||||
// thing a later turn can read, since the audio is gone -- must carry the
|
||||
// transcript, not the frontend's "[voice message]" note.
|
||||
func TestTurnRunnerTranscribesSpeechOnTheModelOwningSide(t *testing.T) {
|
||||
store := NewSessionStoreMemory()
|
||||
router := NewRouter(nil, nil, "telegram")
|
||||
router.Identity = "archie"
|
||||
router.InitSessions(store)
|
||||
|
||||
prepared := &turnTestPreparedModel{reply: "answer"}
|
||||
transcriber := &fakeTranscriber{text: "turn on the lights"}
|
||||
runner := NewTurnRunner(TurnRunnerConfig{
|
||||
Router: router,
|
||||
Sessions: store,
|
||||
Models: &fakeModelManager{models: []string{"test/model"}, activeModel: "test/model"},
|
||||
Personas: NewPersonaRegistry(DefaultPersonas()),
|
||||
Model: &turnTestModel{prepared: prepared},
|
||||
Transcriber: transcriber,
|
||||
BotUser: "archie",
|
||||
Channel: "telegram",
|
||||
Operator: "Sam",
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
})
|
||||
|
||||
voiceBytes := []byte("ogg-voice-bytes")
|
||||
_, err := runner.Run(context.Background(), Inbound{
|
||||
Message: messaging.Message{
|
||||
SourceID: "src-voice",
|
||||
ConversationID: messaging.ConversationID{ChannelID: "chat-voice"},
|
||||
Sender: "user",
|
||||
Role: messaging.RoleUser,
|
||||
Text: "[voice message]",
|
||||
},
|
||||
Media: []messaging.MediaAttachment{{Type: messaging.MediaTypeVoice, MIMEType: "audio/ogg", Data: voiceBytes}},
|
||||
}, DeltaFunc(func(string) {}))
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
|
||||
if transcriber.calls != 1 {
|
||||
t.Fatalf("Transcribe calls = %d, want 1", transcriber.calls)
|
||||
}
|
||||
if string(transcriber.audio) != string(voiceBytes) {
|
||||
t.Errorf("Transcribe received %q, want the attachment bytes carried across the boundary", transcriber.audio)
|
||||
}
|
||||
history, err := store.RecentMessages(context.Background(), "chat-voice", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages() error = %v", err)
|
||||
}
|
||||
if len(history) == 0 {
|
||||
t.Fatal("no stored history; the inbound turn was not recorded")
|
||||
}
|
||||
want := "[voice transcription]\nturn on the lights"
|
||||
if history[0].Text != want {
|
||||
t.Errorf("stored inbound text = %q, want %q", history[0].Text, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTurnRunnerKeepsTheNoteWhenTranscriptionDegrades pins the required
|
||||
// degradation: a nil Transcriber, a failed call, or an empty transcript leaves
|
||||
// the frontend's media note in place rather than failing the turn.
|
||||
func TestTurnRunnerKeepsTheNoteWhenTranscriptionDegrades(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
transcriber messaging.Transcriber
|
||||
}{
|
||||
{name: "no transcriber configured"},
|
||||
{name: "transcriber failed", transcriber: &fakeTranscriber{err: errors.New("provider down")}},
|
||||
{name: "empty transcript", transcriber: &fakeTranscriber{text: " "}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
store := NewSessionStoreMemory()
|
||||
router := NewRouter(nil, nil, "telegram")
|
||||
router.Identity = "archie"
|
||||
router.InitSessions(store)
|
||||
|
||||
runner := NewTurnRunner(TurnRunnerConfig{
|
||||
Router: router,
|
||||
Sessions: store,
|
||||
Models: &fakeModelManager{models: []string{"test/model"}, activeModel: "test/model"},
|
||||
Personas: NewPersonaRegistry(DefaultPersonas()),
|
||||
Model: &turnTestModel{prepared: &turnTestPreparedModel{reply: "answer"}},
|
||||
Transcriber: tc.transcriber,
|
||||
BotUser: "archie",
|
||||
Channel: "telegram",
|
||||
Operator: "Sam",
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
})
|
||||
|
||||
_, err := runner.Run(context.Background(), Inbound{
|
||||
Message: messaging.Message{
|
||||
SourceID: "src-voice-" + tc.name,
|
||||
ConversationID: messaging.ConversationID{ChannelID: "chat-voice-degrade"},
|
||||
Sender: "user",
|
||||
Role: messaging.RoleUser,
|
||||
Text: "[voice message]",
|
||||
},
|
||||
Media: []messaging.MediaAttachment{{Type: messaging.MediaTypeVoice, MIMEType: "audio/ogg", Data: []byte("ogg-voice-bytes")}},
|
||||
}, DeltaFunc(func(string) {}))
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
history, err := store.RecentMessages(context.Background(), "chat-voice-degrade", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages() error = %v", err)
|
||||
}
|
||||
if len(history) == 0 || history[0].Text != "[voice message]" {
|
||||
t.Fatalf("stored inbound text = %#v, want the media note preserved", history)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestTurnRunnerDoesNotTranscribePlainAudio keeps the capability scoped to
|
||||
// speech: a forwarded music file reaches the model as an audio attachment, not
|
||||
// a transcription call.
|
||||
func TestTurnRunnerDoesNotTranscribePlainAudio(t *testing.T) {
|
||||
store := NewSessionStoreMemory()
|
||||
router := NewRouter(nil, nil, "telegram")
|
||||
router.Identity = "archie"
|
||||
router.InitSessions(store)
|
||||
|
||||
transcriber := &fakeTranscriber{text: "should not be used"}
|
||||
runner := NewTurnRunner(TurnRunnerConfig{
|
||||
Router: router,
|
||||
Sessions: store,
|
||||
Models: &fakeModelManager{models: []string{"test/model"}, activeModel: "test/model"},
|
||||
Personas: NewPersonaRegistry(DefaultPersonas()),
|
||||
Model: &turnTestModel{prepared: &turnTestPreparedModel{reply: "answer"}},
|
||||
Transcriber: transcriber,
|
||||
BotUser: "archie",
|
||||
Channel: "telegram",
|
||||
Operator: "Sam",
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
})
|
||||
|
||||
_, err := runner.Run(context.Background(), Inbound{
|
||||
Message: messaging.Message{
|
||||
SourceID: "src-audio",
|
||||
ConversationID: messaging.ConversationID{ChannelID: "chat-audio"},
|
||||
Sender: "user",
|
||||
Role: messaging.RoleUser,
|
||||
Text: "[audio]",
|
||||
},
|
||||
Media: []messaging.MediaAttachment{{Type: messaging.MediaTypeAudio, MIMEType: "audio/mpeg", Data: []byte("mp3-bytes")}},
|
||||
}, DeltaFunc(func(string) {}))
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if transcriber.calls != 0 {
|
||||
t.Errorf("Transcribe calls = %d, want 0 for non-speech audio", transcriber.calls)
|
||||
}
|
||||
history, err := store.RecentMessages(context.Background(), "chat-audio", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages() error = %v", err)
|
||||
}
|
||||
if len(history) == 0 || history[0].Text != "[audio]" {
|
||||
t.Fatalf("stored inbound text = %#v, want the plain audio note", history)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user