diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 109c2ce15..574dc4280 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -41,6 +41,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/health" "github.com/samcharles93/archie-core/internal/domain/identity" domainmemory "github.com/samcharles93/archie-core/internal/domain/memory" + "github.com/samcharles93/archie-core/internal/domain/messaging" "github.com/samcharles93/archie-core/internal/domain/scheduling" "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow" @@ -61,6 +62,7 @@ import ( "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/infrastructure/taskactions" "github.com/samcharles93/archie-core/internal/infrastructure/toolbuilder" + "github.com/samcharles93/archie-core/internal/infrastructure/transcription" "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/plugin" "github.com/samcharles93/archie-core/internal/plugin/pluginextract" @@ -254,6 +256,15 @@ type boot struct { // unrelated call. embeddings domainembedding.Client + // transcriber is the optional voice-transcription capability. It is built + // beside the chat runtime from this process's own [models]/[providers], + // which is the model-owning side of the Messaging boundary: a channel + // frontend carries a voice note's bytes across the inbound wire, and this + // process turns them into text before the turn is recorded. Nil when no + // role is configured or the provider credential did not resolve; a turn + // then keeps the frontend's media note rather than failing. + transcriber messaging.Transcriber + toolReg *tools.Registry chatModels *chatModelManager personas *gateway.PersonaRegistry @@ -872,6 +883,26 @@ func (b *boot) setupEmbeddings(cfg config.Config, log *slog.Logger) { } } +// setupTranscriber builds the optional voice-transcription capability. It is +// the model-owning side of the Messaging Service boundary: the channel +// frontend carries a voice note's audio bytes across the inbound wire but +// holds no provider credential, so this process turns them into text before +// the turn's message is recorded. Like setupEmbeddings, a client is wired only +// when models["transcription"] names a provider/model and that provider's +// credential resolves; a configured-but-unusable role is logged so the +// degradation is visible, while a role never configured is silent. +func (b *boot) setupTranscriber(cfg config.Config, log *slog.Logger) { + client, ok := transcription.New(cfg.Models, cfg.Providers, transcription.Options{ + ResolveSecret: b.secrets.Resolve, + }) + if ok { + b.transcriber = client + log.Info("voice transcription enabled", "role", transcription.Role) + } else if cfg.Models[transcription.Role] != "" { + log.Warn("voice transcription configured but unavailable; capability disabled", "role", transcription.Role) + } +} + // setupLLMAndChat wires the runtime, tool registry, model management, // personas and the dashboard's chat service. func (b *boot) setupLLMAndChat(ctx context.Context) error { diff --git a/internal/app/archied/chat_setup.go b/internal/app/archied/chat_setup.go index 6b6a76f4d..a8410eaa3 100644 --- a/internal/app/archied/chat_setup.go +++ b/internal/app/archied/chat_setup.go @@ -14,6 +14,7 @@ import ( "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/daemon" + "github.com/samcharles93/archie-core/internal/domain/messaging" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/gateway" "github.com/samcharles93/archie-core/internal/installtype" @@ -63,6 +64,12 @@ type chatSetup struct { // process makes, which the /status health source reads back. Nil disables // recording (the recorder's methods tolerate that rather than panicking). ProviderOutcomes *providerOutcomeRecorder + // Transcriber is the voice-transcription capability built by + // setupChatRuntime from this process's own [models]/[providers]. Every turn + // runner carries it so a speech attachment is transcribed before its + // message is recorded. Nil disables the capability (no role configured, or + // the provider credential did not resolve), leaving the frontend's note. + Transcriber messaging.Transcriber } // daemonRunningVersions reports the component versions this process can @@ -190,6 +197,7 @@ func newChatTurnRunner( MemoryEngine: s.MemoryEngine, MemoryWriter: s.MemoryWriter, UserIdentity: userIdentityResolver(), + Transcriber: s.Transcriber, Log: s.Log, }) if err := runner.Recover(ctx); err != nil && s.Log != nil { diff --git a/internal/app/archied/gateway_runtime.go b/internal/app/archied/gateway_runtime.go index 0c675516d..54d902740 100644 --- a/internal/app/archied/gateway_runtime.go +++ b/internal/app/archied/gateway_runtime.go @@ -25,6 +25,11 @@ func (b *boot) setupChatRuntime(ctx context.Context, cfg config.Config) error { // non-command message processing. providers := executionProviders(cfg) b.setLLM(agentexec.NewRuntime(providers)) + // Transcription is a model-role capability, like the chat models it sits + // beside: it is built here, on the model-owning side, from this process's + // own [models]/[providers]. The Messaging Service carries a voice note's + // bytes to the Gateway and never holds the credential. + b.setupTranscriber(cfg, b.log) b.toolReg = tools.NewRegistry() chatModels := newChatModelManager(cfg.Models, cfg.Chat.Models) catalog, catalogModels := b.catalogState() @@ -142,6 +147,9 @@ func (b *boot) setupGatewayChat(ctx context.Context, actor gateway.ChatTaskActor Bus: b.bus, Log: b.log, MemoryEngine: b.memoryStore(), MemoryWriter: b.memoryWriter(), + // The transcription capability is built beside the chat runtime, from + // the model-role configuration only this process reads. + Transcriber: b.transcriber, // The Gateway executes the web chat's turns, so it is the process // that must record their outcomes for /status. ProviderOutcomes: b.providerOutcomes, diff --git a/internal/app/archied/transcriber_test.go b/internal/app/archied/transcriber_test.go new file mode 100644 index 000000000..2f5d827d5 --- /dev/null +++ b/internal/app/archied/transcriber_test.go @@ -0,0 +1,39 @@ +package archied + +import ( + "log/slog" + "testing" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/infrastructure/transcription" + "github.com/samcharles93/archie-core/internal/secret" +) + +// TestSetupTranscriberBuildsOnTheModelOwningSide pins where the voice +// capability is constructed: beside the chat runtime, from the process's own +// [models].transcription and [providers.*]. The Messaging Service carries the +// audio bytes across the inbound wire but no longer builds this client. +func TestSetupTranscriberBuildsOnTheModelOwningSide(t *testing.T) { + log := slog.New(slog.DiscardHandler) + + t.Run("configured role wires a transcriber", func(t *testing.T) { + b := &boot{secrets: secret.NewRegistry()} + b.setupTranscriber(config.Config{ + Models: map[string]string{transcription.Role: "local/whisper-large-v3"}, + Providers: map[string]config.Provider{ + "local": {Class: "openai-compatible", BaseURL: "http://127.0.0.1:8080/v1"}, + }, + }, log) + if b.transcriber == nil { + t.Fatal("transcriber = nil, want a client built from [models].transcription") + } + }) + + t.Run("absent role degrades to nil", func(t *testing.T) { + b := &boot{secrets: secret.NewRegistry()} + b.setupTranscriber(config.Config{}, log) + if b.transcriber != nil { + t.Fatalf("transcriber = %v, want nil when no role is configured", b.transcriber) + } + }) +} diff --git a/internal/app/archiemessaging/compose.go b/internal/app/archiemessaging/compose.go index f1681db75..8380a6e18 100644 --- a/internal/app/archiemessaging/compose.go +++ b/internal/app/archiemessaging/compose.go @@ -82,7 +82,6 @@ func compose(ctx context.Context, d deps) (*Service, error) { } tg := telegram.New(d.Config.TelegramToken, d.Config.Telegram.AllowedUserIDs, d.Log) tg.Settings = d.Settings - tg.Transcriber = d.Config.Transcriber configureTelegram(ctx, tg, d.Config, d.Chat, d.Log) if err := srv.add("telegram", tg, telegramValidateConfigMap(d.Config.Telegram)); err != nil { return nil, err diff --git a/internal/app/archiemessaging/config.go b/internal/app/archiemessaging/config.go index 72f73a5e1..1d1883e9e 100644 --- a/internal/app/archiemessaging/config.go +++ b/internal/app/archiemessaging/config.go @@ -8,9 +8,7 @@ import ( "os" "github.com/samcharles93/archie-core/internal/config" - "github.com/samcharles93/archie-core/internal/domain/messaging" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" - "github.com/samcharles93/archie-core/internal/infrastructure/transcription" "github.com/samcharles93/archie-core/internal/secret" ) @@ -32,11 +30,6 @@ type ResolvedConfig struct { BotUser string HealthURL string ShowToolCalls bool - // Transcriber is the optional voice-transcription capability. Nil means it - // is not configured (or its provider did not resolve), and a voice note - // keeps its media note. Constructed from [models].transcription and the - // matching [providers.*] entry. - Transcriber messaging.Transcriber } type projection struct { @@ -50,8 +43,6 @@ type projection struct { botUser string healthURL string showToolCalls bool - models map[string]string - providers map[string]config.Provider } // Resolve loads the configuration, extracts the messaging projection, resolves @@ -101,7 +92,6 @@ func Resolve(o Options, log *slog.Logger) (ResolvedConfig, error) { BotUser: proj.botUser, HealthURL: proj.healthURL, ShowToolCalls: proj.showToolCalls, - Transcriber: setupTranscriber(proj, secrets, log), }, nil } @@ -131,8 +121,6 @@ func project(cfg config.Config) projection { botUser: cfg.BotUser, healthURL: cfg.Health.URL(), showToolCalls: cfg.Chat.ShowToolCalls, - models: cfg.Models, - providers: cfg.Providers, } } @@ -177,20 +165,3 @@ func resolveWebhookSecret(route config.WebhookRoute, secrets *secret.Registry) ( } return "", nil } - -// setupTranscriber builds the optional voice-transcription capability. A -// client is wired only when [models].transcription names a provider/model and -// that provider's credential resolves; a configured-but-unusable role is -// logged so the degradation is visible, while a role that was never -// configured is silent (mirrors archied's setupEmbeddings). -func setupTranscriber(proj projection, secrets *secret.Registry, log *slog.Logger) messaging.Transcriber { - client, ok := transcription.New(proj.models, proj.providers, transcription.Options{ResolveSecret: secrets.Resolve}) - if ok { - log.Info("voice transcription enabled", "role", transcription.Role) - return client - } - if proj.models[transcription.Role] != "" { - log.Warn("voice transcription configured but unavailable; capability disabled", "role", transcription.Role) - } - return nil -} diff --git a/internal/app/archiemessaging/config_test.go b/internal/app/archiemessaging/config_test.go index b03c72869..11834f5b4 100644 --- a/internal/app/archiemessaging/config_test.go +++ b/internal/app/archiemessaging/config_test.go @@ -72,70 +72,6 @@ allowed_user_ids = [12345] } } -func TestResolveBuildsTranscriberFromModelRole(t *testing.T) { - write := func(t *testing.T, extra string) string { - t.Helper() - tmp := t.TempDir() - path := filepath.Join(tmp, "config.toml") - content := ` -bot_user = "testbot" - -[services.gateway] -target = "127.0.0.1:8999" -` + extra - if err := os.WriteFile(path, []byte(content), 0o600); err != nil { - t.Fatal(err) - } - return path - } - - t.Run("configured role wires a transcriber", func(t *testing.T) { - path := write(t, ` -[models] -transcription = "local/whisper-large-v3" - -[providers.local] -class = "openai-compatible" -base_url = "http://127.0.0.1:8080/v1" -`) - resolved, err := Resolve(Options{Config: path}, slog.Default()) - if err != nil { - t.Fatalf("Resolve: %v", err) - } - if resolved.Transcriber == nil { - t.Error("Transcriber = nil, want a client built from [models].transcription") - } - }) - - t.Run("absent role degrades to nil", func(t *testing.T) { - resolved, err := Resolve(Options{Config: write(t, "")}, slog.Default()) - if err != nil { - t.Fatalf("Resolve: %v", err) - } - if resolved.Transcriber != nil { - t.Errorf("Transcriber = %v, want nil when no role is configured", resolved.Transcriber) - } - }) - - t.Run("configured but unusable role degrades to nil", func(t *testing.T) { - path := write(t, ` -[models] -transcription = "openai/whisper-1" - -[providers.openai] -class = "openai" -api_key_env = "TRANSCRIPTION_TEST_UNSET_KEY" -`) - resolved, err := Resolve(Options{Config: path}, slog.Default()) - if err != nil { - t.Fatalf("Resolve: %v", err) - } - if resolved.Transcriber != nil { - t.Errorf("Transcriber = %v, want nil when the credential is missing", resolved.Transcriber) - } - }) -} - func TestServiceStartAndStop(t *testing.T) { d := deps{ Config: ResolvedConfig{ diff --git a/internal/channels/telegram/inboundmedia.go b/internal/channels/telegram/inboundmedia.go index dc2dab4ea..f6ab34661 100644 --- a/internal/channels/telegram/inboundmedia.go +++ b/internal/channels/telegram/inboundmedia.go @@ -21,9 +21,6 @@ type inboundMedia struct { // note is a bracketed, self-describing placeholder such as "[photo]" // or "[document: report.pdf]". note string - // transcribe marks audio a configured Transcriber should turn into text - // (a Telegram voice note). False leaves the note as-is. - transcribe bool } // extractInboundMedia maps a Telegram message onto the attachment it @@ -73,12 +70,12 @@ func extractInboundMedia(msg *models.Message) (inboundMedia, bool) { case msg.Voice != nil: return inboundMedia{attachment: messaging.MediaAttachment{ - Type: "audio", + Type: messaging.MediaTypeVoice, FileID: msg.Voice.FileID, MIMEType: msg.Voice.MimeType, FileSize: nilIfZero(msg.Voice.FileSize), Duration: &msg.Voice.Duration, - }, note: "[voice message]", transcribe: true}, true + }, note: "[voice message]"}, true } return inboundMedia{}, false } diff --git a/internal/channels/telegram/inboundmedia_test.go b/internal/channels/telegram/inboundmedia_test.go index 2fb4f9026..3c736f0d8 100644 --- a/internal/channels/telegram/inboundmedia_test.go +++ b/internal/channels/telegram/inboundmedia_test.go @@ -158,7 +158,7 @@ func TestExtractInboundMediaMapsTelegramKinds(t *testing.T) { msg: &models.Message{ Voice: &models.Voice{FileID: "voice1", MimeType: "audio/ogg", Duration: 8}, }, - wantType: "audio", + wantType: messaging.MediaTypeVoice, wantFileID: "voice1", wantMIME: "audio/ogg", wantNote: "[voice message]", @@ -333,24 +333,6 @@ func TestDefaultHandlerSilentForNonContentMessages(t *testing.T) { } } -// fakeTranscriber records the audio it was handed and returns a fixed -// transcript, so a test can assert both that transcription ran on the -// downloaded bytes and what the turn then carried. -type fakeTranscriber struct { - text string - err error - calls int - audio []byte -} - -var _ messaging.Transcriber = (*fakeTranscriber)(nil) - -func (f *fakeTranscriber) Transcribe(_ context.Context, audio []byte) (string, error) { - f.calls++ - f.audio = append([]byte(nil), audio...) - return f.text, f.err -} - // voiceUpdate is a Telegram voice note from the authorized sender. func voiceUpdate(fileID string) *models.Message { return &models.Message{ @@ -362,11 +344,12 @@ func voiceUpdate(fileID string) *models.Message { } } -// TestDefaultHandlerTranscribesVoiceNoteIntoText drives the behaviour this -// feature exists for: a configured Transcriber turns the downloaded voice -// note into the turn's text, behind a provenance marker so the agent knows -// it is reading a machine transcript. -func TestDefaultHandlerTranscribesVoiceNoteIntoText(t *testing.T) { +// TestDefaultHandlerMarksVoiceNoteAsSpeech drives the frontend half of the +// boundary: the messaging process downloads the note, classifies it as speech +// (messaging.MediaTypeVoice), and carries the bytes and the note to the turn. +// Turning those bytes into text is the model-owning process's job -- this +// process holds no provider credential -- so no transcription happens here. +func TestDefaultHandlerMarksVoiceNoteAsSpeech(t *testing.T) { voiceBytes := []byte("ogg-voice-bytes") b, _ := newRecordedAPI(t, func(w http.ResponseWriter, r *http.Request) bool { if strings.HasPrefix(r.URL.Path, "/file/bot1:test/") { @@ -386,72 +369,30 @@ func TestDefaultHandlerTranscribesVoiceNoteIntoText(t *testing.T) { return ch, nil }, } - transcriber := &fakeTranscriber{text: "turn the lights on"} g := testMediaGateway(b) - g.Transcriber = transcriber g.defaultHandler(client)(context.Background(), b, &models.Update{Message: voiceUpdate("voice1")}) select { case in := <-inboundCh: - want := "[voice transcription]\nturn the lights on" - if in.Message.Text != want { - t.Errorf("Text = %q, want %q", in.Message.Text, want) + if len(in.Media) != 1 || in.Media[0].Type != messaging.MediaTypeVoice { + t.Fatalf("inbound media = %#v, want the voice attachment marked as speech", in.Media) } - if len(in.Media) != 1 || in.Media[0].Type != "audio" { - t.Fatalf("inbound media = %#v, want the voice attachment", in.Media) + if string(in.Media[0].Data) != string(voiceBytes) { + t.Errorf("inbound media bytes = %q, want the downloaded voice bytes", in.Media[0].Data) } - if transcriber.calls != 1 { - t.Fatalf("Transcribe calls = %d, want 1", transcriber.calls) - } - if string(transcriber.audio) != string(voiceBytes) { - t.Errorf("Transcribe received %q, want the downloaded voice bytes", transcriber.audio) + if in.Message.Text != "[voice message]" { + t.Errorf("Text = %q, want the media note preserved for the model-owning side", in.Message.Text) } case <-time.After(5 * time.Second): t.Fatal("voice note never reached the chat turn") } } -// TestDefaultHandlerKeepsVoiceNoteWhenTranscriptionUnavailable pins the -// degradation the config contract requires: with no Transcriber the voice -// note still becomes a turn, carrying today's "[voice message]" note rather -// than failing or going silent. -func TestDefaultHandlerKeepsVoiceNoteWhenTranscriptionUnavailable(t *testing.T) { - b, _ := newRecordedAPI(t, func(w http.ResponseWriter, r *http.Request) bool { - if strings.HasPrefix(r.URL.Path, "/file/bot1:test/") { - _, _ = w.Write([]byte("ogg-voice-bytes")) - return true - } - return false - }) - - inboundCh := make(chan messaging.Inbound, 1) - client := &fakeChatContract{ - streamFunc: func(_ context.Context, in messaging.Inbound) (<-chan messaging.ChatEvent, error) { - inboundCh <- in - ch := make(chan messaging.ChatEvent, 1) - ch <- messaging.ChatEvent{Kind: "done", Text: "done", SessionID: "s1"} - close(ch) - return ch, nil - }, - } - g := testMediaGateway(b) // Transcriber left nil on purpose - - g.defaultHandler(client)(context.Background(), b, &models.Update{Message: voiceUpdate("voice1")}) - - select { - case in := <-inboundCh: - if in.Message.Text != "[voice message]" { - t.Errorf("Text = %q, want the media note preserved", in.Message.Text) - } - case <-time.After(5 * time.Second): - t.Fatal("voice note was dropped when transcription was unavailable") - } -} - -// TestDefaultHandlerDoesNotTranscribePlainAudio keeps the flag scoped to -// voice notes: a forwarded music file must not spend a transcription call. -func TestDefaultHandlerDoesNotTranscribePlainAudio(t *testing.T) { +// TestDefaultHandlerMarksPlainAudioAsAudio keeps the speech distinction: a +// forwarded music file is ordinary audio, so the model-owning side knows not +// to spend a transcription call on it. +func TestDefaultHandlerMarksPlainAudioAsAudio(t *testing.T) { b, _ := newRecordedAPI(t, func(w http.ResponseWriter, r *http.Request) bool { if strings.HasPrefix(r.URL.Path, "/file/bot1:test/") { _, _ = w.Write([]byte("mp3-bytes")) @@ -470,9 +411,7 @@ func TestDefaultHandlerDoesNotTranscribePlainAudio(t *testing.T) { return ch, nil }, } - transcriber := &fakeTranscriber{text: "should not be used"} g := testMediaGateway(b) - g.Transcriber = transcriber g.defaultHandler(client)(context.Background(), b, &models.Update{Message: &models.Message{ ID: 12, @@ -484,12 +423,12 @@ func TestDefaultHandlerDoesNotTranscribePlainAudio(t *testing.T) { select { case in := <-inboundCh: + if len(in.Media) != 1 || in.Media[0].Type != messaging.MediaTypeAudio { + t.Fatalf("inbound media = %#v, want the plain audio attachment", in.Media) + } if in.Message.Text != "[audio]" { t.Errorf("Text = %q, want the plain audio note", in.Message.Text) } - if transcriber.calls != 0 { - t.Errorf("Transcribe calls = %d, want 0 for non-voice audio", transcriber.calls) - } case <-time.After(5 * time.Second): t.Fatal("audio update never reached the chat turn") } diff --git a/internal/channels/telegram/telegram.go b/internal/channels/telegram/telegram.go index 575ace7ad..0feb98cb7 100644 --- a/internal/channels/telegram/telegram.go +++ b/internal/channels/telegram/telegram.go @@ -50,12 +50,6 @@ type Gateway struct { Updates UpdateService // Settings executes control-plane commands in this authenticated adapter. Settings *messaging.SettingsCommand - // Transcriber turns voice notes into text before their turn is dispatched. - // Nil means the capability is not configured (or its provider did not - // resolve): a voice note then keeps its "[voice message]" note, exactly - // the behaviour before transcription existed. The composition root owns - // construction, so this adapter never holds a provider credential itself. - Transcriber messaging.Transcriber // ResolveActor maps a channel-native sender to an authenticated Archie // identity. Nil deliberately permits reads but makes writes fail closed. ResolveActor func(int64) (string, bool) @@ -655,7 +649,6 @@ func (g *Gateway) submitTurn(ctx context.Context, b *bot.Bot, msg *models.Messag if !isTurn { return } - transcribeVoice := hasMedia && media.transcribe // The lane key must be the session, so that /stop -- which resolves // the same key -- reaches the turn the sender is actually watching. @@ -665,7 +658,6 @@ func (g *Gateway) submitTurn(ctx context.Context, b *bot.Bot, msg *models.Messag if !g.fetchTurnMedia(turnCtx, b, &gm, chatID, threadID) { return } - g.transcribeVoiceNote(turnCtx, &gm, msg.Caption, transcribeVoice) // If the turn invokes a tool that requires human approval, // the dispatch layer blocks on this approver. Nil is fine @@ -776,27 +768,6 @@ func (g *Gateway) fetchTurnMedia(ctx context.Context, b *bot.Bot, gm *messaging. return true } -// transcribeVoiceNote replaces a voice note's stored placeholder with the -// transcript, so the agent reads what was said rather than "[voice message]". -// It degrades: a nil Transcriber, an error, or an empty transcript leaves the -// placeholder in place -- a turn must never fail because transcription was -// unavailable. -func (g *Gateway) transcribeVoiceNote(ctx context.Context, gm *messaging.Inbound, caption string, wanted bool) { - if !wanted || g.Transcriber == nil || len(gm.Media) == 0 { - return - } - transcript, err := g.Transcriber.Transcribe(ctx, gm.Media[0].Data) - if err != nil { - g.log.Warn("voice transcription failed; keeping the media note", "error", err) - return - } - if strings.TrimSpace(transcript) == "" { - g.log.Info("voice transcription produced no text; keeping the media note") - return - } - gm.Message.Text = messaging.TranscribedText(transcript, caption) -} - // ── helpers ────────────────────────────────────────────────────── // conversationID addresses the chat a Telegram message arrived in: the diff --git a/internal/domain/messaging/media.go b/internal/domain/messaging/media.go index 02939e284..d2bcbbaa3 100644 --- a/internal/domain/messaging/media.go +++ b/internal/domain/messaging/media.go @@ -9,6 +9,21 @@ package messaging // attachment is accepted rather than rejected as an oversized message. const MaxInboundAttachmentBytes = 20 << 20 +// Type values. MediaTypeVoice is speech the sender recorded -- a Telegram +// voice note -- as distinct from MediaTypeAudio, a music or generic audio +// file. The distinction is what tells the model-owning turn runner to +// transcribe a speech attachment before the turn: a voice note's bytes reach +// the Gateway, which owns the provider credential, but the Gateway must not +// spend a transcription call on a forwarded song. The remaining values are +// what they say. +const ( + MediaTypeImage = "image" + MediaTypeVideo = "video" + MediaTypeAudio = "audio" + MediaTypeDocument = "document" + MediaTypeVoice = "voice" +) + // MediaAttachment describes a file attached to a message. Platform-agnostic; // platform-specific fields live in Raw. // @@ -16,7 +31,7 @@ const MaxInboundAttachmentBytes = 20 << 20 // "unknown" across a JSON round-trip: a 0-byte file is valid and // different from a file whose size was never set. type MediaAttachment struct { - // Type is "image", "video", "audio", or "document". + // Type is "image", "video", "audio", "voice", or "document". Type string `json:"type"` // FileID is the platform-assigned identifier for download APIs. diff --git a/internal/domain/messaging/transcribe.go b/internal/domain/messaging/transcribe.go index aa95b78ad..053335f60 100644 --- a/internal/domain/messaging/transcribe.go +++ b/internal/domain/messaging/transcribe.go @@ -51,3 +51,33 @@ func TranscribedText(transcript, caption string) string { } return strings.Join(parts, "\n") } + +// TranscribedMessageText renders the text a transcribed speech attachment +// contributes to a message, preserving any caption the sender attached. +// +// messageText is the inbound text a channel frontend rendered for an +// attachment: the media note alone, or the note followed by the sender's +// caption on the next line. The note is a bracketed, self-describing +// placeholder (see internal/channels/telegram's turnMessageText); it is +// replaced by the provenance marker and transcript, and anything after it +// survives as the caption. Text that carries no recognisable note is treated +// as a caption in full, so nothing the sender wrote is dropped. +// +// The transcription happens on the model-owning side of the Messaging +// boundary, which receives the audio bytes but not the caption as a separate +// field; this is the function that turns the frontend's rendered text into +// the transcript the turn is recorded with. +func TranscribedMessageText(messageText, transcript string) string { + note, caption, _ := strings.Cut(messageText, "\n") + if !isMediaNote(note) { + return TranscribedText(transcript, messageText) + } + return TranscribedText(transcript, caption) +} + +// isMediaNote reports whether s is a channel frontend's bracketed media +// placeholder such as "[voice message]". +func isMediaNote(s string) bool { + s = strings.TrimSpace(s) + return len(s) >= 2 && s[0] == '[' && s[len(s)-1] == ']' +} diff --git a/internal/domain/messaging/transcribe_test.go b/internal/domain/messaging/transcribe_test.go index 7978e23f6..9bb03c718 100644 --- a/internal/domain/messaging/transcribe_test.go +++ b/internal/domain/messaging/transcribe_test.go @@ -2,6 +2,45 @@ package messaging import "testing" +// TestTranscribedMessageTextReplacesTheMediaNote pins the boundary handoff: +// the frontend renders a voice note as its bracketed note (plus any caption), +// and the model-owning side only has that text when it transcribes. The note +// must be replaced by the transcript; the caption must survive. +func TestTranscribedMessageTextReplacesTheMediaNote(t *testing.T) { + cases := []struct { + name string + messageText string + transcript string + want string + }{ + { + name: "note alone is replaced", + messageText: "[voice message]", + transcript: "hello there", + want: "[voice transcription]\nhello there", + }, + { + name: "caption after the note survives", + messageText: "[voice message]\nignore the background noise", + transcript: "what time is it", + want: "[voice transcription]\nwhat time is it\nignore the background noise", + }, + { + name: "text with no note is kept as the caption", + messageText: "typed it by hand", + transcript: "spoken words", + want: "[voice transcription]\nspoken words\ntyped it by hand", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := TranscribedMessageText(tc.messageText, tc.transcript); got != tc.want { + t.Errorf("TranscribedMessageText(%q, %q) = %q, want %q", tc.messageText, tc.transcript, got, tc.want) + } + }) + } +} + // TestTranscribedTextMarksProvenanceAndKeepsCaption pins the stored shape of // a transcribed voice note: history is text-only and the original bytes are // gone, so the marker is the only thing that tells a later reader the words diff --git a/internal/gateway/turn.go b/internal/gateway/turn.go index 3022c4d6d..00befd31a 100644 --- a/internal/gateway/turn.go +++ b/internal/gateway/turn.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log/slog" + "strings" "sync" "time" @@ -87,11 +88,20 @@ const defaultTurnPersistenceTimeout = 5 * time.Second // runner. The runner is channel-neutral apart from the channel name used in // prompts, session tools, and completion events. type TurnRunnerConfig struct { - Router *Router - Sessions SessionStore - Models ModelManager - Personas PersonaPromptSource - Model TurnModel + Router *Router + Sessions SessionStore + Models ModelManager + Personas PersonaPromptSource + Model TurnModel + // Transcriber turns an inbound speech attachment (messaging.MediaTypeVoice) + // into text before the turn's inbound message is recorded, so stored + // history reads what was said rather than the frontend's note. The + // composition root builds it from the process's own [models]/[providers], + // which only the model-owning side of the Messaging boundary reads; a + // channel frontend carries the attachment bytes across the inbound wire + // but never holds the provider credential. Nil keeps the note (the + // capability is optional and degrades). + Transcriber messaging.Transcriber Ledger TurnLedger OwnerID string PersistenceTimeout time.Duration @@ -485,6 +495,13 @@ func (r *TurnRunner) Run(ctx context.Context, in Inbound, stream TurnStream) (st return replayed, err } + // A message already recorded by an earlier attempt (turn.InputMessageID + // set) keeps the transcript that attempt persisted; transcribing again + // would duplicate the call and could disagree with stored history. + if turn.InputMessageID == "" { + in.Message = r.transcribeSpeech(ctx, in) + } + in.Message, history, err = r.recordInboundMessage(ctx, &turn, sessionID, in.Message, history) if err != nil { return "", r.failTurn(ctx, turn, err) @@ -497,6 +514,39 @@ func (r *TurnRunner) Run(ctx context.Context, in Inbound, stream TurnStream) (st return r.generateAndComplete(ctx, turn, sessionID, prep, stream) } +// transcribeSpeech replaces an inbound speech attachment's placeholder note +// with the transcript before the turn's message is recorded, so the store -- +// and every later turn reading its history -- keeps the words rather than a +// note whose audio is gone. It degrades: a nil Transcriber, a failed call, or +// an empty transcript leaves the frontend's note in place. A turn must never +// fail because transcription was unavailable. +func (r *TurnRunner) transcribeSpeech(ctx context.Context, in Inbound) messaging.Message { + msg := in.Message + if r.Transcriber == nil { + return msg + } + for _, att := range in.Media { + if att.Type != messaging.MediaTypeVoice || len(att.Data) == 0 { + continue + } + transcript, err := r.Transcriber.Transcribe(ctx, att.Data) + if err != nil { + if r.Log != nil { + r.Log.Warn("voice transcription failed; keeping the media note", "error", err) + } + continue + } + if strings.TrimSpace(transcript) == "" { + if r.Log != nil { + r.Log.Info("voice transcription produced no text; keeping the media note") + } + continue + } + msg.Text = messaging.TranscribedMessageText(msg.Text, transcript) + } + return msg +} + // generateAndComplete runs the model, persists the assistant message and // completed turn, and publishes the turn-completed event. func (r *TurnRunner) generateAndComplete( diff --git a/internal/gateway/turn_media_test.go b/internal/gateway/turn_media_test.go index 8e04d2a08..2eb0a98b9 100644 --- a/internal/gateway/turn_media_test.go +++ b/internal/gateway/turn_media_test.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "log/slog" "reflect" "testing" @@ -54,3 +55,184 @@ func TestTurnRunnerGivesInboundMediaToTheModelRequest(t *testing.T) { t.Errorf("model request media = %#v, want %#v", prepared.lastRequest.Media, media) } } + +// fakeTranscriber records the audio it is handed and returns a fixed +// transcript, so a test can prove the model-owning turn runner transcribed +// the bytes a channel frontend carried across the boundary. +type fakeTranscriber struct { + text string + err error + calls int + audio []byte +} + +func (f *fakeTranscriber) Transcribe(_ context.Context, audio []byte) (string, error) { + f.calls++ + f.audio = append([]byte(nil), audio...) + return f.text, f.err +} + +// TestTurnRunnerTranscribesSpeechOnTheModelOwningSide drives the behaviour the +// boundary move exists for: a voice attachment's bytes reach the process that +// owns the model and provider credentials, which turns them into the turn's +// text before the inbound message is recorded. The stored history -- the only +// thing a later turn can read, since the audio is gone -- must carry the +// transcript, not the frontend's "[voice message]" note. +func TestTurnRunnerTranscribesSpeechOnTheModelOwningSide(t *testing.T) { + store := NewSessionStoreMemory() + router := NewRouter(nil, nil, "telegram") + router.Identity = "archie" + router.InitSessions(store) + + prepared := &turnTestPreparedModel{reply: "answer"} + transcriber := &fakeTranscriber{text: "turn on the lights"} + runner := NewTurnRunner(TurnRunnerConfig{ + Router: router, + Sessions: store, + Models: &fakeModelManager{models: []string{"test/model"}, activeModel: "test/model"}, + Personas: NewPersonaRegistry(DefaultPersonas()), + Model: &turnTestModel{prepared: prepared}, + Transcriber: transcriber, + BotUser: "archie", + Channel: "telegram", + Operator: "Sam", + Log: slog.New(slog.DiscardHandler), + }) + + voiceBytes := []byte("ogg-voice-bytes") + _, err := runner.Run(context.Background(), Inbound{ + Message: messaging.Message{ + SourceID: "src-voice", + ConversationID: messaging.ConversationID{ChannelID: "chat-voice"}, + Sender: "user", + Role: messaging.RoleUser, + Text: "[voice message]", + }, + Media: []messaging.MediaAttachment{{Type: messaging.MediaTypeVoice, MIMEType: "audio/ogg", Data: voiceBytes}}, + }, DeltaFunc(func(string) {})) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + + if transcriber.calls != 1 { + t.Fatalf("Transcribe calls = %d, want 1", transcriber.calls) + } + if string(transcriber.audio) != string(voiceBytes) { + t.Errorf("Transcribe received %q, want the attachment bytes carried across the boundary", transcriber.audio) + } + history, err := store.RecentMessages(context.Background(), "chat-voice", 10) + if err != nil { + t.Fatalf("RecentMessages() error = %v", err) + } + if len(history) == 0 { + t.Fatal("no stored history; the inbound turn was not recorded") + } + want := "[voice transcription]\nturn on the lights" + if history[0].Text != want { + t.Errorf("stored inbound text = %q, want %q", history[0].Text, want) + } +} + +// TestTurnRunnerKeepsTheNoteWhenTranscriptionDegrades pins the required +// degradation: a nil Transcriber, a failed call, or an empty transcript leaves +// the frontend's media note in place rather than failing the turn. +func TestTurnRunnerKeepsTheNoteWhenTranscriptionDegrades(t *testing.T) { + cases := []struct { + name string + transcriber messaging.Transcriber + }{ + {name: "no transcriber configured"}, + {name: "transcriber failed", transcriber: &fakeTranscriber{err: errors.New("provider down")}}, + {name: "empty transcript", transcriber: &fakeTranscriber{text: " "}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store := NewSessionStoreMemory() + router := NewRouter(nil, nil, "telegram") + router.Identity = "archie" + router.InitSessions(store) + + runner := NewTurnRunner(TurnRunnerConfig{ + Router: router, + Sessions: store, + Models: &fakeModelManager{models: []string{"test/model"}, activeModel: "test/model"}, + Personas: NewPersonaRegistry(DefaultPersonas()), + Model: &turnTestModel{prepared: &turnTestPreparedModel{reply: "answer"}}, + Transcriber: tc.transcriber, + BotUser: "archie", + Channel: "telegram", + Operator: "Sam", + Log: slog.New(slog.DiscardHandler), + }) + + _, err := runner.Run(context.Background(), Inbound{ + Message: messaging.Message{ + SourceID: "src-voice-" + tc.name, + ConversationID: messaging.ConversationID{ChannelID: "chat-voice-degrade"}, + Sender: "user", + Role: messaging.RoleUser, + Text: "[voice message]", + }, + Media: []messaging.MediaAttachment{{Type: messaging.MediaTypeVoice, MIMEType: "audio/ogg", Data: []byte("ogg-voice-bytes")}}, + }, DeltaFunc(func(string) {})) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + history, err := store.RecentMessages(context.Background(), "chat-voice-degrade", 10) + if err != nil { + t.Fatalf("RecentMessages() error = %v", err) + } + if len(history) == 0 || history[0].Text != "[voice message]" { + t.Fatalf("stored inbound text = %#v, want the media note preserved", history) + } + }) + } +} + +// TestTurnRunnerDoesNotTranscribePlainAudio keeps the capability scoped to +// speech: a forwarded music file reaches the model as an audio attachment, not +// a transcription call. +func TestTurnRunnerDoesNotTranscribePlainAudio(t *testing.T) { + store := NewSessionStoreMemory() + router := NewRouter(nil, nil, "telegram") + router.Identity = "archie" + router.InitSessions(store) + + transcriber := &fakeTranscriber{text: "should not be used"} + runner := NewTurnRunner(TurnRunnerConfig{ + Router: router, + Sessions: store, + Models: &fakeModelManager{models: []string{"test/model"}, activeModel: "test/model"}, + Personas: NewPersonaRegistry(DefaultPersonas()), + Model: &turnTestModel{prepared: &turnTestPreparedModel{reply: "answer"}}, + Transcriber: transcriber, + BotUser: "archie", + Channel: "telegram", + Operator: "Sam", + Log: slog.New(slog.DiscardHandler), + }) + + _, err := runner.Run(context.Background(), Inbound{ + Message: messaging.Message{ + SourceID: "src-audio", + ConversationID: messaging.ConversationID{ChannelID: "chat-audio"}, + Sender: "user", + Role: messaging.RoleUser, + Text: "[audio]", + }, + Media: []messaging.MediaAttachment{{Type: messaging.MediaTypeAudio, MIMEType: "audio/mpeg", Data: []byte("mp3-bytes")}}, + }, DeltaFunc(func(string) {})) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if transcriber.calls != 0 { + t.Errorf("Transcribe calls = %d, want 0 for non-speech audio", transcriber.calls) + } + history, err := store.RecentMessages(context.Background(), "chat-audio", 10) + if err != nil { + t.Fatalf("RecentMessages() error = %v", err) + } + if len(history) == 0 || history[0].Text != "[audio]" { + t.Fatalf("stored inbound text = %#v, want the plain audio note", history) + } +}