diff --git a/docs/data/generated/contracts.json b/docs/data/generated/contracts.json index 7618bfde8..8a359db66 100644 --- a/docs/data/generated/contracts.json +++ b/docs/data/generated/contracts.json @@ -312,6 +312,11 @@ "additionalProperties": false, "description": "MediaAttachment describes a file attached to a message.", "properties": { + "data": { + "contentEncoding": "base64", + "description": "Data carries the attachment bytes in-process. An inbound path sets\nit after downloading the file; outbound senders never set it and\nread Path or URL instead. It never crosses a wire or persistence\nboundary -- it is a turn-scoped value, not a stored one.", + "type": "string" + }, "duration": { "description": "Duration is the play time in seconds for audio and video.", "type": "integer" diff --git a/internal/app/archied/chat_turn_model.go b/internal/app/archied/chat_turn_model.go index feccb4e5c..a341f9f4a 100644 --- a/internal/app/archied/chat_turn_model.go +++ b/internal/app/archied/chat_turn_model.go @@ -100,6 +100,80 @@ func (m *preparedChatTurnModel) ToolSummaries() []gateway.ToolSummary { return append([]gateway.ToolSummary(nil), m.toolInfo...) } +// buildTurnMessages maps the gateway's compressed history onto ai-sdk chat +// messages. The turn's inbound media is not part of stored history, so it +// is merged here into the final user message as content parts: the model +// sees the caption text plus each attachment's bytes (or URL), exactly for +// the turn it arrived in. +func buildTurnMessages(request gateway.TurnModelRequest) []chat.Message { + messages := make([]chat.Message, len(request.Messages)) + for i, message := range request.Messages { + role := chat.RoleUser + switch message.Role { + case "assistant": + role = chat.RoleAssistant + case "system": + role = chat.RoleSystem + } + messages[i] = chat.Message{Role: role, Content: message.Content} + } + parts := mediaParts(request.Media) + if len(parts) == 0 { + return messages + } + last := len(messages) - 1 + // The media belongs to the inbound message, which is always the final + // user entry; if the view ever ends differently, content stays intact + // over silently attaching parts to an unrelated message. + if last < 0 || messages[last].Role != chat.RoleUser { + return messages + } + userMessages := messages[last] + userMessages.Parts = append(chat.Parts{chat.TextPart{Text: userMessages.Content}}, parts...) + userMessages.Content = "" + messages[last] = userMessages + return messages +} + +// mediaParts converts inbound media attachments into ai-sdk content +// parts: images get ImagePart, everything else FilePart. An attachment +// carrying neither bytes nor a URL names nothing consumable and is +// dropped -- the turn's text still reaches the model. +func mediaParts(media []gateway.MediaAttachment) []chat.Part { + if len(media) == 0 { + return nil + } + parts := make([]chat.Part, 0, len(media)) + for _, att := range media { + if att.Type == "image" { + switch { + case len(att.Data) > 0 && att.MIMEType != "": + parts = append(parts, chat.ImagePart{Data: att.Data, MediaType: att.MIMEType}) + case att.URL != "": + parts = append(parts, chat.ImagePart{URL: att.URL}) + } + continue + } + part := chat.FilePart{Name: att.FileName, MediaType: att.MIMEType} + if len(att.Data) > 0 { + part.Data = att.Data + } else { + part.URL = att.URL + } + if len(part.Data) == 0 && part.URL == "" { + continue + } + if part.MediaType == "" { + part.MediaType = "application/octet-stream" + } + parts = append(parts, part) + } + if len(parts) == 0 { + return nil + } + return parts +} + func (m *preparedChatTurnModel) ToolSchemaTokens() int { return m.toolTokens } @@ -110,17 +184,7 @@ func (m *preparedChatTurnModel) Generate( stream gateway.TurnStream, ) (string, error) { options := m.options - options.Messages = make([]chat.Message, len(request.Messages)) - for i, message := range request.Messages { - role := chat.RoleUser - switch message.Role { - case "assistant": - role = chat.RoleAssistant - case "system": - role = chat.RoleSystem - } - options.Messages[i] = chat.Message{Role: role, Content: message.Content} - } + options.Messages = buildTurnMessages(request) // This is one provider response's output allowance, not a turn- // continuation budget. Tool loops remain free to continue. options.MaxTokens = request.MaxOutputTokens diff --git a/internal/app/archied/chat_turn_model_media_test.go b/internal/app/archied/chat_turn_model_media_test.go new file mode 100644 index 000000000..5cd169569 --- /dev/null +++ b/internal/app/archied/chat_turn_model_media_test.go @@ -0,0 +1,116 @@ +package archied + +import ( + "testing" + + "github.com/samcharles93/ai-sdk/chat" + + "github.com/samcharles93/archie-core/internal/gateway" +) + +// TestBuildTurnMessagesAttachesMediaPartsToTheFinalUserMessage: inbound +// media only reaches the model when the turn's request is turned into +// message parts; the attachment belongs on the message the user just sent. +func TestBuildTurnMessagesAttachesMediaPartsToTheFinalUserMessage(t *testing.T) { + request := gateway.TurnModelRequest{ + Messages: []gateway.CompressedMessage{ + {Role: "system", Content: "sys"}, + {Role: "user", Content: "what is this?"}, + }, + Media: []gateway.MediaAttachment{ + {Type: "image", MIMEType: "image/png", Data: []byte("png-bytes")}, + {Type: "image", URL: "https://example.invalid/pic.jpg"}, + {Type: "document", MIMEType: "application/pdf", FileName: "report.pdf", Data: []byte("pdf-bytes")}, + }, + } + + messages := buildTurnMessages(request) + if len(messages) != 2 { + t.Fatalf("messages = %d, want 2", len(messages)) + } + if messages[0].Content != "sys" || len(messages[0].Parts) != 0 { + t.Errorf("system message = %#v, want plain content and no parts", messages[0]) + } + if messages[1].Content != "" { + t.Errorf("user content = %q, want it moved into a text part", messages[1].Content) + } + parts := messages[1].Parts + if len(parts) != 4 { + t.Fatalf("user parts = %d, want text plus three media parts: %#v", len(parts), parts) + } + if text, ok := parts[0].(chat.TextPart); !ok || text.Text != "what is this?" { + t.Errorf("part 0 = %#v, want the caption text", parts[0]) + } + image, ok := parts[1].(chat.ImagePart) + if !ok || string(image.Data) != "png-bytes" || image.MediaType != "image/png" { + t.Errorf("part 1 = %#v, want an image part carrying the png bytes", parts[1]) + } + linked, ok := parts[2].(chat.ImagePart) + if !ok || linked.URL != "https://example.invalid/pic.jpg" { + t.Errorf("part 2 = %#v, want an image part pointing at the URL", parts[2]) + } + file, ok := parts[3].(chat.FilePart) + if !ok { + t.Fatalf("part 3 = %#v, want a file part", parts[3]) + } + if string(file.Data) != "pdf-bytes" || file.MediaType != "application/pdf" || file.Name != "report.pdf" { + t.Errorf("file part = %#v, want the pdf bytes with type and name", file) + } +} + +// TestBuildTurnMessagesSkipsUndeliverableMedia: an attachment carrying +// neither bytes nor a URL names nothing the model can consume, so it must +// be dropped rather than sent as an empty part. +func TestBuildTurnMessagesSkipsUndeliverableMedia(t *testing.T) { + request := gateway.TurnModelRequest{ + Messages: []gateway.CompressedMessage{ + {Role: "user", Content: "look"}, + }, + Media: []gateway.MediaAttachment{ + {Type: "video"}, + {Type: "image", MIMEType: "image/png", Data: []byte("png")}, + }, + } + messages := buildTurnMessages(request) + fileSeen := false + for _, p := range messages[0].Parts { + if _, ok := p.(chat.FilePart); ok { + fileSeen = true + } + } + if fileSeen { + t.Errorf("parts = %#v, want the empty attachment dropped", messages[0].Parts) + } + if len(messages[0].Parts) != 2 { + t.Fatalf("user parts = %d, want text plus the one deliverable image", len(messages[0].Parts)) + } + if text, ok := messages[0].Parts[0].(chat.TextPart); !ok || text.Text != "look" { + t.Errorf("part 0 = %#v, want the text part", messages[0].Parts[0]) + } +} + +// TestBuildTurnMessagesWithoutMediaKeepsPlainContent is the regression +// guard for the overwhelmingly common case: text-only turns stay exactly as +// they were, with Content and no parts. +func TestBuildTurnMessagesWithoutMediaKeepsPlainContent(t *testing.T) { + request := gateway.TurnModelRequest{ + Messages: []gateway.CompressedMessage{ + {Role: "system", Content: "sys"}, + {Role: "user", Content: "plain"}, + {Role: "assistant", Content: "reply"}, + }, + } + messages := buildTurnMessages(request) + for i, want := range []struct { + role chat.Role + content string + }{ + {chat.RoleSystem, "sys"}, + {chat.RoleUser, "plain"}, + {chat.RoleAssistant, "reply"}, + } { + if messages[i].Content != want.content || messages[i].Parts != nil { + t.Errorf("message %d = %#v, want content %q and no parts", i, messages[i], want.content) + } + } +} diff --git a/internal/channels/telegram/inboundmedia.go b/internal/channels/telegram/inboundmedia.go new file mode 100644 index 000000000..51ace7f4c --- /dev/null +++ b/internal/channels/telegram/inboundmedia.go @@ -0,0 +1,208 @@ +package telegram + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + + "github.com/go-telegram/bot" + "github.com/go-telegram/bot/models" + + "github.com/samcharles93/archie-core/internal/domain/messaging" +) + +// inboundMedia pairs an inbound attachment with the textual note the +// transcript stores in its place: stored history is text-only, so the +// note is what later turns read once the attachment's bytes are gone. +type inboundMedia struct { + attachment messaging.MediaAttachment + // note is a bracketed, self-describing placeholder such as "[photo]" + // or "[document: report.pdf]". + note string +} + +// extractInboundMedia maps a Telegram message onto the attachment it +// carries. A Telegram message carries at most one file (a photo is the +// set of its resolutions and collapses into one image), so the boolean is +// false exactly for messages with nothing to download. +func extractInboundMedia(msg *models.Message) (inboundMedia, bool) { + switch { + case len(msg.Photo) > 0: + att := messaging.MediaAttachment{Type: "image", MIMEType: "image/jpeg"} + for _, size := range msg.Photo { + if largestPhoto(size, att) { + att.FileID = size.FileID + att.Width, att.Height = &size.Width, &size.Height + if size.FileSize != 0 { + att.FileSize = new(int64(size.FileSize)) + } + } + } + return inboundMedia{att, "[photo]"}, true + + case msg.Document != nil: + return inboundMedia{messaging.MediaAttachment{ + Type: "document", + FileID: msg.Document.FileID, + MIMEType: msg.Document.MimeType, + FileName: msg.Document.FileName, + FileSize: nilIfZero(msg.Document.FileSize), + }, documentNote(msg.Document.FileName)}, true + + case msg.Video != nil: + return inboundMedia{videoAttachment(msg.Video.FileID, msg.Video.MimeType, nilIfZero(msg.Video.FileSize), &msg.Video.Width, &msg.Video.Height, &msg.Video.Duration), "[video]"}, true + + case msg.VideoNote != nil: + size := int64(msg.VideoNote.FileSize) + return inboundMedia{videoAttachment(msg.VideoNote.FileID, "video/mp4", nilIfZero(size), nil, nil, &msg.VideoNote.Duration), "[video message]"}, true + + case msg.Audio != nil: + return inboundMedia{messaging.MediaAttachment{ + Type: "audio", + FileID: msg.Audio.FileID, + MIMEType: msg.Audio.MimeType, + FileName: msg.Audio.FileName, + FileSize: nilIfZero(msg.Audio.FileSize), + Duration: &msg.Audio.Duration, + }, "[audio]"}, true + + case msg.Voice != nil: + return inboundMedia{messaging.MediaAttachment{ + Type: "audio", + FileID: msg.Voice.FileID, + MIMEType: msg.Voice.MimeType, + FileSize: nilIfZero(msg.Voice.FileSize), + Duration: &msg.Voice.Duration, + }, "[voice message]"}, true + } + return inboundMedia{}, false +} + +// largestPhoto reports whether candidate is larger than the best size +// recorded so far in att's width/height pair. +func largestPhoto(candidate models.PhotoSize, att messaging.MediaAttachment) bool { + if att.Width == nil { + return true + } + candidateArea, bestArea := candidate.Width*candidate.Height, *att.Width**att.Height + return candidateArea > bestArea +} + +func videoAttachment(fileID, mimeType string, fileSize *int64, width, height, duration *int) messaging.MediaAttachment { + return messaging.MediaAttachment{ + Type: "video", FileID: fileID, MIMEType: mimeType, + FileSize: fileSize, Width: width, Height: height, Duration: duration, + } +} + +func documentNote(name string) string { + if name == "" { + return "[document]" + } + return fmt.Sprintf("[document: %s]", name) +} + +func nilIfZero(v int64) *int64 { + if v == 0 { + return nil + } + return &v +} + +// maxInboundDownloadBytes is the Bot API download ceiling: bots cannot +// fetch files larger than 20 MB through getFile, so a larger attachment +// must be refused with a clear notice rather than met with a Bot API 400 +// nobody reads. +var maxInboundDownloadBytes int64 = 20 * 1024 * 1024 + +var errInboundMediaTooLarge = errors.New("file exceeds the bot download limit") + +// downloadMedia turns a Telegram file id into the file's bytes: getFile +// resolves the transfer path, then the file is fetched from the Bot API +// server. Called on the turn lane, which is already running the rest of +// the download-and-chat work for this message. +func (g *Gateway) downloadMedia(ctx context.Context, b *bot.Bot, att messaging.MediaAttachment) ([]byte, error) { + if att.FileID == "" { + return nil, fmt.Errorf("media attachment carries no file id") + } + if att.FileSize != nil && *att.FileSize > maxInboundDownloadBytes { + return nil, fmt.Errorf("%w: %d bytes", errInboundMediaTooLarge, *att.FileSize) + } + file, err := b.GetFile(ctx, &bot.GetFileParams{FileID: att.FileID}) + if err != nil { + return nil, fmt.Errorf("resolve telegram file: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, b.FileDownloadLink(file), nil) + if err != nil { + return nil, fmt.Errorf("download telegram file: %w", err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("download telegram file: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("download telegram file: status %d", resp.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxInboundDownloadBytes+1)) + if err != nil { + return nil, fmt.Errorf("download telegram file: %w", err) + } + if int64(len(data)) > maxInboundDownloadBytes { + return nil, errInboundMediaTooLarge + } + return data, nil +} + +// turnMessageText renders the text an inbound message's turn is built +// from: plain text as received, or the attachment note plus caption when +// the message carries a file. The note is what later turns read from +// history, so a captioned photo whose caption alone would read as the +// whole message ("what is this?") still names its attachment. +func turnMessageText(msg *models.Message, media inboundMedia, hasMedia bool) string { + if !hasMedia { + return msg.Text + } + if msg.Caption == "" { + return media.note + } + return media.note + "\n" + msg.Caption +} + +// unsupportedMediaNotice returns the refusal for content kinds Archie +// cannot read, and "" for update content that is not a message to answer +// at all -- a group member change or pin must stay silent, while a +// sticker the sender meant Archie to react to must not disappear into the +// void. +func unsupportedMediaNotice(msg *models.Message) string { + switch { + case msg.Sticker != nil: + return "I can't read stickers — send the image or file directly and I'll take a look." + case msg.Location != nil: + return "I can't use locations yet." + case msg.Contact != nil: + return "I can't use contact cards yet." + case msg.Poll != nil: + return "I can't use polls yet." + } + return "" +} + +// mediaReplyLabel names an attachment the turn could not serve, in the +// sentence the sender reads. +func mediaReplyLabel(att messaging.MediaAttachment) string { + switch { + case att.Type == "image": + return "I received your photo " + case att.Type == "document" && att.FileName != "": + return fmt.Sprintf("I received %s ", att.FileName) + case att.Type == "document": + return "I received your file " + case att.Type == "video": + return "I received your video " + default: + return "I received your audio attachment " + } +} diff --git a/internal/channels/telegram/inboundmedia_test.go b/internal/channels/telegram/inboundmedia_test.go new file mode 100644 index 000000000..51ae49844 --- /dev/null +++ b/internal/channels/telegram/inboundmedia_test.go @@ -0,0 +1,383 @@ +package telegram + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/go-telegram/bot" + "github.com/go-telegram/bot/models" + + "github.com/samcharles93/archie-core/internal/domain/messaging" +) + +// recordedAPI captures every Bot API request so a test can assert a reply +// was sent (or, equally, that none was). +type recordedAPI struct { + mu sync.Mutex + calls []string // "path body" lines, one per request +} + +func newRecordedAPI(t *testing.T, fileRoute func(w http.ResponseWriter, r *http.Request) bool) (*bot.Bot, *recordedAPI) { + t.Helper() + rec := &recordedAPI{} + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if fileRoute != nil && fileRoute(w, r) { + return + } + body := readBody(r) + rec.mu.Lock() + rec.calls = append(rec.calls, r.URL.Path+" "+body) + rec.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "getFile") { + _, _ = w.Write([]byte(`{"ok":true,"result":{"file_path":"photos/pic.jpg"}}`)) + return + } + _, _ = w.Write([]byte(`{"ok":true,"result":true}`)) + })) + t.Cleanup(api.Close) + + b, err := bot.New("1:test", bot.WithServerURL(api.URL), bot.WithSkipGetMe()) + if err != nil { + t.Fatalf("new test bot: %v", err) + } + return b, rec +} + +func readBody(r *http.Request) string { + _ = r.ParseMultipartForm(1 << 20) + if r.PostForm != nil { + return r.PostForm.Encode() + } + return r.URL.RawQuery +} + +func (rec *recordedAPI) contains(fragment string) bool { + rec.mu.Lock() + defer rec.mu.Unlock() + for _, call := range rec.calls { + if strings.Contains(strings.ToLower(call), strings.ToLower(fragment)) { + return true + } + } + return false +} + +func (rec *recordedAPI) callCount() int { + rec.mu.Lock() + defer rec.mu.Unlock() + return len(rec.calls) +} + +func testMediaGateway(b *bot.Bot) *Gateway { + g := New("1:test", []int64{42}, slog.New(slog.DiscardHandler)) + g.bot = b + g.turns = messaging.NewTurns(g.log) + return g +} + +// TestExtractInboundMediaMapsTelegramKinds walks every supported message +// kind. A message's media decides the file the bot must download, so a +// mapping error sends the wrong file or sends nothing at all. +func TestExtractInboundMediaMapsTelegramKinds(t *testing.T) { + cases := []struct { + name string + msg *models.Message + wantType string + wantFileID string + wantMIME string + wantName string + wantNote string + wantNoMedia bool + }{ + { + name: "photo picks the largest size", + msg: &models.Message{ + Photo: []models.PhotoSize{ + {FileID: "small", Width: 90, Height: 80}, + {FileID: "big", Width: 1280, Height: 960, FileSize: 150000}, + }, + }, + wantType: "image", + wantFileID: "big", + wantMIME: "image/jpeg", + wantNote: "[photo]", + }, + { + name: "document", + msg: &models.Message{ + Document: &models.Document{ + FileID: "doc1", FileName: "report.pdf", MimeType: "application/pdf", FileSize: 2048, + }, + }, + wantType: "document", + wantFileID: "doc1", + wantMIME: "application/pdf", + wantName: "report.pdf", + wantNote: "[document: report.pdf]", + }, + { + name: "video", + msg: &models.Message{ + Video: &models.Video{FileID: "vid1", Duration: 12, MimeType: "video/mp4"}, + }, + wantType: "video", + wantFileID: "vid1", + wantMIME: "video/mp4", + wantNote: "[video]", + }, + { + name: "video note", + msg: &models.Message{ + VideoNote: &models.VideoNote{FileID: "vnote1", Duration: 30}, + }, + wantType: "video", + wantFileID: "vnote1", + wantMIME: "video/mp4", + wantNote: "[video message]", + }, + { + name: "audio", + msg: &models.Message{ + Audio: &models.Audio{FileID: "aud1", FileName: "song.mp3", MimeType: "audio/mpeg", Duration: 200}, + }, + wantType: "audio", + wantFileID: "aud1", + wantMIME: "audio/mpeg", + wantName: "song.mp3", + wantNote: "[audio]", + }, + { + name: "voice", + msg: &models.Message{ + Voice: &models.Voice{FileID: "voice1", MimeType: "audio/ogg", Duration: 8}, + }, + wantType: "audio", + wantFileID: "voice1", + wantMIME: "audio/ogg", + wantNote: "[voice message]", + }, + { + name: "plain text message carries no media", + msg: &models.Message{Text: "hello"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := extractInboundMedia(tc.msg) + if tc.wantType == "" { + if ok { + t.Fatalf("extractInboundMedia(%s) = ok, want none", tc.name) + } + return + } + if !ok { + t.Fatalf("extractInboundMedia(%s) missed, want %s", tc.name, tc.wantType) + } + att := got.attachment + if att.Type != tc.wantType { + t.Errorf("Type = %q, want %q", att.Type, tc.wantType) + } + if att.FileID != tc.wantFileID { + t.Errorf("FileID = %q, want %q", att.FileID, tc.wantFileID) + } + if att.MIMEType != tc.wantMIME { + t.Errorf("MIMEType = %q, want %q", att.MIMEType, tc.wantMIME) + } + if att.FileName != tc.wantName { + t.Errorf("FileName = %q, want %q", att.FileName, tc.wantName) + } + if got.note != tc.wantNote { + t.Errorf("note = %q, want %q", got.note, tc.wantNote) + } + }) + } +} + +// TestTurnMessageTextKeepsCaptionWithPlaceholder: history is text-only, so +// the stored turn text must name the attachment and keep any caption. +func TestTurnMessageTextKeepsCaptionWithPlaceholder(t *testing.T) { + media, ok := extractInboundMedia(&models.Message{ + Photo: []models.PhotoSize{{FileID: "a", Width: 10, Height: 10}}, + Caption: "what is this?", + }) + if !ok { + t.Fatal("media missed") + } + got := turnMessageText(&models.Message{Text: "", Caption: "what is this?"}, media, ok) + want := "[photo]\nwhat is this?" + if got != want { + t.Errorf("turnMessageText = %q, want %q", got, want) + } +} + +// TestDefaultHandlerSubmitsPhotoTurnWithDownloadedMedia drives the one +// behaviour at the heart of the gap: a photo update reaches the chat turn +// with the file's bytes attached, not as a dropped message. +func TestDefaultHandlerSubmitsPhotoTurnWithDownloadedMedia(t *testing.T) { + photoBytes := []byte("jpeg-bytes") + var fileServed bool + b, rec := newRecordedAPI(t, func(w http.ResponseWriter, r *http.Request) bool { + if strings.HasPrefix(r.URL.Path, "/file/bot1:test/") { + w.Header().Set("Content-Type", "image/jpeg") + _, _ = w.Write(photoBytes) + fileServed = true + return true + } + return false + }) + + inboundCh := make(chan messaging.Inbound, 1) + client := &fakeChatContract{ + streamFunc: func(_ context.Context, in messaging.Inbound) (<-chan messaging.ChatEvent, error) { + inboundCh <- in + ch := make(chan messaging.ChatEvent, 2) + ch <- messaging.ChatEvent{Kind: "started", SessionID: "s1"} + ch <- messaging.ChatEvent{Kind: "done", Text: "done", SessionID: "s1"} + close(ch) + return ch, nil + }, + } + g := testMediaGateway(b) + + handler := g.defaultHandler(client) + handler(context.Background(), b, &models.Update{Message: &models.Message{ + ID: 5, + Date: 1, + From: &models.User{ID: 42, Username: "sam"}, + Chat: models.Chat{ID: 7, Type: "private"}, + Photo: []models.PhotoSize{{FileID: "big", Width: 1280, Height: 960, FileSize: 150000}}, + Caption: "look at this", + }}) + + select { + case in := <-inboundCh: + if len(in.Media) != 1 { + t.Fatalf("inbound media = %#v, want one attachment", in.Media) + } + att := in.Media[0] + if att.Type != "image" || att.MIMEType != "image/jpeg" || att.FileID != "big" { + t.Errorf("attachment = %#v, want image with the photo's file id", att) + } + if string(att.Data) != string(photoBytes) { + t.Errorf("Data = %q, want the downloaded photo bytes", att.Data) + } + want := "[photo]\nlook at this" + if in.Message.Text != want { + t.Errorf("Text = %q, want %q", in.Message.Text, want) + } + if in.Message.SourceID != "5" { + t.Errorf("SourceID = %q, want the Telegram message id", in.Message.SourceID) + } + if !fileServed { + t.Error("download never fetched the file bytes") + } + case <-time.After(5 * time.Second): + t.Fatalf("photo update never reached the chat turn; api calls were:\n%s", + strings.Join(rec.calls, "\n")) + } +} + +// TestDefaultHandlerRepliesToUnsupportedMedia keeps the gap from returning +// silently in the other direction: content types Archie cannot read get a +// clear notice rather than the void. +func TestDefaultHandlerRepliesToUnsupportedMedia(t *testing.T) { + cases := []struct { + name string + setup func(*models.Message) + reply string + }{ + {"sticker", func(m *models.Message) { m.Sticker = &models.Sticker{FileID: "s"} }, "sticker"}, + {"location", func(m *models.Message) { m.Location = &models.Location{} }, "location"}, + {"contact", func(m *models.Message) { m.Contact = &models.Contact{} }, "contact"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b, rec := newRecordedAPI(t, nil) + g := testMediaGateway(b) + msg := &models.Message{ + ID: 6, + Date: 1, + From: &models.User{ID: 42, Username: "sam"}, + Chat: models.Chat{ID: 7, Type: "private"}, + } + tc.setup(msg) + g.defaultHandler(&fakeChatContract{})(context.Background(), b, &models.Update{Message: msg}) + if !rec.contains(tc.reply) { + t.Errorf("no reply mentioning %q was sent; calls:\n%s", tc.reply, strings.Join(rec.calls, "\n")) + } + }) + } +} + +// TestDefaultHandlerSilentForNonContentMessages guards the other update +// kinds a group can carry: no text, no media, no notice -- and so no reply. +func TestDefaultHandlerSilentForNonContentMessages(t *testing.T) { + b, rec := newRecordedAPI(t, nil) + g := testMediaGateway(b) + g.defaultHandler(&fakeChatContract{})(context.Background(), b, &models.Update{Message: &models.Message{ + ID: 7, + Date: 1, + From: &models.User{ID: 42}, + Chat: models.Chat{ID: 7, Type: "private"}, + }}) + if n := rec.callCount(); n != 0 { + t.Errorf("%d API calls for a message with no text, media or notice, want 0", n) + } +} + +// TestDownloadMediaCoversSuccessAndLimits exercises the download the turn +// relies on: the Bot API getFile lookup, the file fetch, and the size gate. +func TestDownloadMediaCoversSuccessAndLimits(t *testing.T) { + fileBytes := []byte("file-bytes") + b, rec := newRecordedAPI(t, func(w http.ResponseWriter, r *http.Request) bool { + if strings.HasPrefix(r.URL.Path, "/file/bot1:test/") { + _, _ = w.Write(fileBytes) + return true + } + return false + }) + g := testMediaGateway(b) + + t.Run("success", func(t *testing.T) { + data, err := g.downloadMedia(context.Background(), b, + messaging.MediaAttachment{Type: "image", FileID: "f1", MIMEType: "image/jpeg"}) + if err != nil { + t.Fatalf("downloadMedia() error = %v", err) + } + if string(data) != string(fileBytes) { + t.Errorf("downloaded %q, want %q", data, fileBytes) + } + if !rec.contains("getfile") { + t.Error("getFile never resolved the file path") + } + }) + + t.Run("oversized declared size is refused before download", func(t *testing.T) { + old := maxInboundDownloadBytes + maxInboundDownloadBytes = 4 + t.Cleanup(func() { maxInboundDownloadBytes = old }) + size := int64(10) + if _, err := g.downloadMedia(context.Background(), b, + messaging.MediaAttachment{Type: "document", FileID: "f2", FileSize: &size}); err == nil { + t.Error("oversized declared size accepted") + } + }) + + t.Run("oversized response body is refused", func(t *testing.T) { + old := maxInboundDownloadBytes + maxInboundDownloadBytes = 4 + t.Cleanup(func() { maxInboundDownloadBytes = old }) + if _, err := g.downloadMedia(context.Background(), b, + messaging.MediaAttachment{Type: "document", FileID: "f3"}); err == nil { + t.Error("response larger than the limit accepted") + } + }) +} diff --git a/internal/channels/telegram/telegram.go b/internal/channels/telegram/telegram.go index ef620b3e4..88eba8cd0 100644 --- a/internal/channels/telegram/telegram.go +++ b/internal/channels/telegram/telegram.go @@ -519,7 +519,7 @@ func (g *Gateway) startHandler() bot.HandlerFunc { } } -// ── default handler (non-command text → router) ───────────── +// ── default handler (commands and non-command messages → router) ───── func (g *Gateway) defaultHandler(client messaging.ChatContract) bot.HandlerFunc { return func(ctx context.Context, b *bot.Bot, update *models.Update) { @@ -529,7 +529,20 @@ func (g *Gateway) defaultHandler(client messaging.ChatContract) bot.HandlerFunc } msg, ok := g.authorizedMessage(ctx, b, update) - if !ok || msg.Text == "" { + if !ok { + return + } + // Media messages arrive before the text gates: a photo has no text, + // so checking it first is what keeps attachments from being dropped. + if _, isMedia := extractInboundMedia(msg); isMedia { + g.submitTurn(ctx, b, msg, client) + return + } + if notice := unsupportedMediaNotice(msg); notice != "" { + g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, notice) + return + } + if msg.Text == "" { return } if isModelSelectorRequest(msg.Text) { @@ -584,27 +597,52 @@ func (g *Gateway) handleCallback(ctx context.Context, b *bot.Bot, update *models // running here would block delivery of every later update -- including the // /stop meant to cancel it, which would sit unread until the turn it was // aimed at had already finished. +// turnInbound renders a Telegram message as the channel-neutral inbound a +// chat turn is built from, with media attachments attached. ok is false +// for messages with neither text nor media, which are not turns at all. +func turnInbound(msg *models.Message) (messaging.Inbound, bool) { + media, hasMedia := extractInboundMedia(msg) + if !hasMedia && msg.Text == "" { + return messaging.Inbound{}, false + } + gm := messaging.Inbound{ + Platform: "telegram", + Message: messaging.Message{ + // Telegram's message ID makes persistence idempotent: the store + // derives a canonical ID from it, so a redelivered update is a + // no-op rather than appending a duplicate or overwriting the + // stored record, which must stay immutable. Date is the sender's + // clock reading and is what history should be ordered by. + SourceID: fmt.Sprintf("%d", msg.ID), + ConversationID: conversationID(msg), + Sender: msg.From.Username, + SenderID: fmt.Sprintf("%d", msg.From.ID), + Role: messaging.RoleUser, + Text: turnMessageText(msg, media, hasMedia), + At: time.Unix(int64(msg.Date), 0).UTC(), + }, + } + if hasMedia { + gm.Media = []messaging.MediaAttachment{media.attachment} + } + return gm, true +} + func (g *Gateway) submitTurn(ctx context.Context, b *bot.Bot, msg *models.Message, client messaging.ChatContract) { - gm := messaging.Inbound{Platform: "telegram", Message: messaging.Message{ - // Telegram's message ID makes persistence idempotent: the store - // derives a canonical ID from it, so a redelivered update is a - // no-op rather than appending a duplicate or overwriting the - // stored record, which must stay immutable. Date is the sender's - // clock reading and is what history should be ordered by. - SourceID: fmt.Sprintf("%d", msg.ID), - ConversationID: conversationID(msg), - Sender: msg.From.Username, - SenderID: fmt.Sprintf("%d", msg.From.ID), - Role: messaging.RoleUser, - Text: msg.Text, - At: time.Unix(int64(msg.Date), 0).UTC(), - }} + gm, isTurn := turnInbound(msg) + if !isTurn { + return + } // The lane key must be the session, so that /stop -- which resolves // the same key -- reaches the turn the sender is actually watching. session := conversationID(msg).String() chatID, threadID := msg.Chat.ID, msg.MessageThreadID g.turns.Submit(ctx, session, func(turnCtx context.Context) { + if !g.fetchTurnMedia(turnCtx, b, &gm, chatID, threadID) { + return + } + // If the turn invokes a tool that requires human approval, // the dispatch layer blocks on this approver. Nil is fine // — most turns need no gating. @@ -629,53 +667,92 @@ func (g *Gateway) submitTurn(ctx context.Context, b *bot.Bot, msg *models.Messag return } - var reply string - var streamErr error - for ev := range events { - switch ev.Kind { - case "delta": - live.Delta(ev.Text) - case "tool": - live.ToolCall(ev.Tool) - case "media": - live.Media(turnCtx, ev.Media) - case "done": - reply = ev.Text - case "error": - streamErr = errors.New(ev.Text) - } - } - stopTyping() - - // A cancelled turn is a /stop, not a fault. The stop handler has - // already acknowledged it, and turnCtx is dead, so there is - // nothing useful left to send from here -- but whatever was - // already streamed stays, minus the cursor that would otherwise - // claim the answer is still being written. abandon strips turnCtx's - // own cancellation before it edits, so the cursor-drop is not itself - // aborted by the /stop that triggered it. - // - // errors.Is(err, context.Canceled) reliably separates a /stop from a - // fault; either way turnCtx is dead by the time we get here. A - // genuine fault is marked as failed rather than left as a clean - // partial: /stop is an acknowledged interruption, a provider error - // is not, and an unmarked partial reads as a finished answer either - // way. - if streamErr != nil || turnCtx.Err() != nil { - if errors.Is(turnCtx.Err(), context.Canceled) || (streamErr != nil && errors.Is(streamErr, context.Canceled)) { - g.log.Info("chat turn stopped", "session", session) - live.abandon(turnCtx) - } else { - g.log.Error("stream turn failed", "error", streamErr) - live.abandonFailed(turnCtx) - } + reply, aborted := g.drainTurnEvents(turnCtx, session, live, stopTyping, events) + if aborted { return } live.finalize(turnCtx, reply) }) } -// ── helpers ────────────────────────────────────────────────── +// drainTurnEvents consumes a streamed turn, rendering deltas, tool calls +// and media as they arrive, and returns the final reply. +// +// A cancelled turn is a /stop, not a fault. The stop handler has already +// acknowledged it, and turnCtx is dead, so there is +// nothing useful left to send from here -- but whatever was +// already streamed stays, minus the cursor that would otherwise +// claim the answer is still being written. abandon strips turnCtx's +// own cancellation before it edits, so the cursor-drop is not itself +// aborted by the /stop that triggered it. +// +// errors.Is(err, context.Canceled) reliably separates a /stop from a +// fault; either way turnCtx is dead by the time we get here. A +// genuine fault is marked as failed rather than left as a clean +// partial: /stop is an acknowledged interruption, a provider error +// is not, and an unmarked partial reads as a finished answer either +// way. +// +// aborted means the turn was abandoned (as a stop or a failure) and its +// reply must not be finalized. +func (g *Gateway) drainTurnEvents( + turnCtx context.Context, session string, + live *liveReply, stopTyping func(), events <-chan messaging.ChatEvent, +) (string, bool) { + var reply string + var streamErr error + for ev := range events { + switch ev.Kind { + case "delta": + live.Delta(ev.Text) + case "tool": + live.ToolCall(ev.Tool) + case "media": + live.Media(turnCtx, ev.Media) + case "done": + reply = ev.Text + case "error": + streamErr = errors.New(ev.Text) + } + } + stopTyping() + + if streamErr != nil || turnCtx.Err() != nil { + if errors.Is(turnCtx.Err(), context.Canceled) || (streamErr != nil && errors.Is(streamErr, context.Canceled)) { + g.log.Info("chat turn stopped", "session", session) + live.abandon(turnCtx) + } else { + g.log.Error("stream turn failed", "error", streamErr) + live.abandonFailed(turnCtx) + } + return "", true + } + return reply, false +} + +// fetchTurnMedia downloads the turn's inbound attachment onto the lane, +// where waiting for the Telegram API belongs: the update worker stays free +// to serve other messages and a /stop while this download progresses. A +// failed download is reported as a failed attachment without starting the +// turn, so the answer does not pretend the media was read -- a later +// turn's history only knows the note, not the picture, and the model must +// not reason from what it never saw. +func (g *Gateway) fetchTurnMedia(ctx context.Context, b *bot.Bot, gm *messaging.Inbound, chatID int64, threadID int) bool { + if len(gm.Media) == 0 || gm.Media[0].Data != nil { + return true + } + data, err := g.downloadMedia(ctx, b, gm.Media[0]) + if err != nil { + g.log.Warn("inbound media download failed", "media_type", gm.Media[0].Type, "error", err) + g.sendMessage(ctx, b, chatID, threadID, + mediaReplyLabel(gm.Media[0])+"but I couldn't download it — try sending it again.") + return false + } + gm.Media[0].Data = data + return true +} + +// ── helpers ────────────────────────────────────────────────────── // conversationID addresses the chat a Telegram message arrived in: the // chat ID, and the topic thread within it for supergroups. diff --git a/internal/contracts/gateway/v1/chat.pb.go b/internal/contracts/gateway/v1/chat.pb.go index 976623572..9764320fa 100644 --- a/internal/contracts/gateway/v1/chat.pb.go +++ b/internal/contracts/gateway/v1/chat.pb.go @@ -161,7 +161,12 @@ type Message struct { // channel, so its own name is "web" even for a Telegram turn, which made // Source.Platform -- the first component of the session natural key -- a // constant. Transport-only, never persisted: the session record keeps it. - Platform string `protobuf:"bytes,10,opt,name=platform,proto3" json:"platform,omitempty"` + Platform string `protobuf:"bytes,10,opt,name=platform,proto3" json:"platform,omitempty"` + // media carries attachment metadata for a message that arrived with a + // file. The attachment's bytes never cross this wire: a stored Media + // with tool_name empty is a sender attachment, and the tool_name on + // tool-produced events stays a stream-only field. + Media []*Media `protobuf:"bytes,11,rep,name=media,proto3" json:"media,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -266,6 +271,13 @@ func (x *Message) GetPlatform() string { return "" } +func (x *Message) GetMedia() []*Media { + if x != nil { + return x.Media + } + return nil +} + type ToolCall struct { state protoimpl.MessageState `protogen:"open.v1"` Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` @@ -2818,7 +2830,7 @@ const file_gateway_v1_chat_proto_rawDesc = "" + "\n" + "created_at\x18\t \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x12@\n" + "\x0elast_active_at\x18\n" + - " \x01(\v2\x1a.google.protobuf.TimestampR\flastActiveAt\"\xa4\x02\n" + + " \x01(\v2\x1a.google.protobuf.TimestampR\flastActiveAt\"\xcd\x02\n" + "\aMessage\x12\x1d\n" + "\n" + "message_id\x18\x01 \x01(\tR\tmessageId\x12\x1b\n" + @@ -2833,7 +2845,8 @@ const file_gateway_v1_chat_proto_rawDesc = "" + "\n" + "budget_key\x18\t \x01(\tR\tbudgetKey\x12\x1a\n" + "\bplatform\x18\n" + - " \x01(\tR\bplatform\"|\n" + + " \x01(\tR\bplatform\x12'\n" + + "\x05media\x18\v \x03(\v2\x11.gateway.v1.MediaR\x05media\"|\n" + "\bToolCall\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + "\x04name\x18\x02 \x01(\tR\x04name\x12\x1e\n" + @@ -3127,74 +3140,75 @@ var file_gateway_v1_chat_proto_depIdxs = []int32{ 50, // 0: gateway.v1.Session.created_at:type_name -> google.protobuf.Timestamp 50, // 1: gateway.v1.Session.last_active_at:type_name -> google.protobuf.Timestamp 50, // 2: gateway.v1.Message.at:type_name -> google.protobuf.Timestamp - 2, // 3: gateway.v1.Turn.tool_calls:type_name -> gateway.v1.ToolCall - 50, // 4: gateway.v1.Turn.created_at:type_name -> google.protobuf.Timestamp - 50, // 5: gateway.v1.Turn.updated_at:type_name -> google.protobuf.Timestamp - 0, // 6: gateway.v1.SnapshotResponse.sessions:type_name -> gateway.v1.Session - 48, // 7: gateway.v1.SnapshotResponse.models_by_provider:type_name -> gateway.v1.SnapshotResponse.ModelsByProviderEntry - 49, // 8: gateway.v1.SnapshotResponse.active_personas:type_name -> gateway.v1.SnapshotResponse.ActivePersonasEntry - 0, // 9: gateway.v1.GetSessionResponse.session:type_name -> gateway.v1.Session - 1, // 10: gateway.v1.RecentMessagesResponse.messages:type_name -> gateway.v1.Message - 4, // 11: gateway.v1.RecentTurnsResponse.turns:type_name -> gateway.v1.Turn - 1, // 12: gateway.v1.RouteRequest.message:type_name -> gateway.v1.Message - 1, // 13: gateway.v1.StreamRequest.message:type_name -> gateway.v1.Message - 2, // 14: gateway.v1.StreamResponse.tool:type_name -> gateway.v1.ToolCall - 3, // 15: gateway.v1.StreamResponse.media:type_name -> gateway.v1.Media - 0, // 16: gateway.v1.SaveSessionRequest.session:type_name -> gateway.v1.Session - 0, // 17: gateway.v1.GetSessionsByChannelResponse.sessions:type_name -> gateway.v1.Session - 0, // 18: gateway.v1.ListSessionsResponse.sessions:type_name -> gateway.v1.Session - 1, // 19: gateway.v1.SaveMessageRequest.message:type_name -> gateway.v1.Message - 1, // 20: gateway.v1.SaveMessagesRequest.messages:type_name -> gateway.v1.Message - 1, // 21: gateway.v1.ReplaceMessagesRequest.messages:type_name -> gateway.v1.Message - 1, // 22: gateway.v1.SearchMessagesResponse.messages:type_name -> gateway.v1.Message - 5, // 23: gateway.v1.SnapshotResponse.ModelsByProviderEntry.value:type_name -> gateway.v1.StringList - 6, // 24: gateway.v1.ChatService.Snapshot:input_type -> gateway.v1.SnapshotRequest - 8, // 25: gateway.v1.ChatService.GetSession:input_type -> gateway.v1.GetSessionRequest - 10, // 26: gateway.v1.ChatService.RecentMessages:input_type -> gateway.v1.RecentMessagesRequest - 12, // 27: gateway.v1.ChatService.RecentTurns:input_type -> gateway.v1.RecentTurnsRequest - 14, // 28: gateway.v1.ChatService.Route:input_type -> gateway.v1.RouteRequest - 16, // 29: gateway.v1.ChatService.Stream:input_type -> gateway.v1.StreamRequest - 18, // 30: gateway.v1.ChatService.Cancel:input_type -> gateway.v1.CancelRequest - 20, // 31: gateway.v1.ChatService.SetPersona:input_type -> gateway.v1.SetPersonaRequest - 22, // 32: gateway.v1.ChatService.ApplyTaskAction:input_type -> gateway.v1.ApplyTaskActionRequest - 24, // 33: gateway.v1.ChatService.ApplyOperatorTaskAction:input_type -> gateway.v1.ApplyOperatorTaskActionRequest - 27, // 34: gateway.v1.ChatService.SaveSession:input_type -> gateway.v1.SaveSessionRequest - 28, // 35: gateway.v1.ChatService.GetSessionsByChannel:input_type -> gateway.v1.GetSessionsByChannelRequest - 36, // 36: gateway.v1.ChatService.DeleteSession:input_type -> gateway.v1.DeleteSessionRequest - 37, // 37: gateway.v1.ChatService.TouchSession:input_type -> gateway.v1.TouchSessionRequest - 38, // 38: gateway.v1.ChatService.ListSessions:input_type -> gateway.v1.ListSessionsRequest - 39, // 39: gateway.v1.ChatService.SaveMessage:input_type -> gateway.v1.SaveMessageRequest - 40, // 40: gateway.v1.ChatService.DeleteRecentMessages:input_type -> gateway.v1.DeleteRecentMessagesRequest - 42, // 41: gateway.v1.ChatService.MessageCount:input_type -> gateway.v1.MessageCountRequest - 44, // 42: gateway.v1.ChatService.SaveMessages:input_type -> gateway.v1.SaveMessagesRequest - 45, // 43: gateway.v1.ChatService.ReplaceMessages:input_type -> gateway.v1.ReplaceMessagesRequest - 46, // 44: gateway.v1.ChatService.SearchMessages:input_type -> gateway.v1.SearchMessagesRequest - 7, // 45: gateway.v1.ChatService.Snapshot:output_type -> gateway.v1.SnapshotResponse - 9, // 46: gateway.v1.ChatService.GetSession:output_type -> gateway.v1.GetSessionResponse - 11, // 47: gateway.v1.ChatService.RecentMessages:output_type -> gateway.v1.RecentMessagesResponse - 13, // 48: gateway.v1.ChatService.RecentTurns:output_type -> gateway.v1.RecentTurnsResponse - 15, // 49: gateway.v1.ChatService.Route:output_type -> gateway.v1.RouteResponse - 17, // 50: gateway.v1.ChatService.Stream:output_type -> gateway.v1.StreamResponse - 19, // 51: gateway.v1.ChatService.Cancel:output_type -> gateway.v1.CancelResponse - 21, // 52: gateway.v1.ChatService.SetPersona:output_type -> gateway.v1.SetPersonaResponse - 23, // 53: gateway.v1.ChatService.ApplyTaskAction:output_type -> gateway.v1.ApplyTaskActionResponse - 25, // 54: gateway.v1.ChatService.ApplyOperatorTaskAction:output_type -> gateway.v1.ApplyOperatorTaskActionResponse - 26, // 55: gateway.v1.ChatService.SaveSession:output_type -> gateway.v1.SaveSessionResponse - 29, // 56: gateway.v1.ChatService.GetSessionsByChannel:output_type -> gateway.v1.GetSessionsByChannelResponse - 31, // 57: gateway.v1.ChatService.DeleteSession:output_type -> gateway.v1.DeleteSessionResponse - 32, // 58: gateway.v1.ChatService.TouchSession:output_type -> gateway.v1.TouchSessionResponse - 30, // 59: gateway.v1.ChatService.ListSessions:output_type -> gateway.v1.ListSessionsResponse - 33, // 60: gateway.v1.ChatService.SaveMessage:output_type -> gateway.v1.SaveMessageResponse - 41, // 61: gateway.v1.ChatService.DeleteRecentMessages:output_type -> gateway.v1.DeleteRecentMessagesResponse - 43, // 62: gateway.v1.ChatService.MessageCount:output_type -> gateway.v1.MessageCountResponse - 34, // 63: gateway.v1.ChatService.SaveMessages:output_type -> gateway.v1.SaveMessagesResponse - 35, // 64: gateway.v1.ChatService.ReplaceMessages:output_type -> gateway.v1.ReplaceMessagesResponse - 47, // 65: gateway.v1.ChatService.SearchMessages:output_type -> gateway.v1.SearchMessagesResponse - 45, // [45:66] is the sub-list for method output_type - 24, // [24:45] is the sub-list for method input_type - 24, // [24:24] is the sub-list for extension type_name - 24, // [24:24] is the sub-list for extension extendee - 0, // [0:24] is the sub-list for field type_name + 3, // 3: gateway.v1.Message.media:type_name -> gateway.v1.Media + 2, // 4: gateway.v1.Turn.tool_calls:type_name -> gateway.v1.ToolCall + 50, // 5: gateway.v1.Turn.created_at:type_name -> google.protobuf.Timestamp + 50, // 6: gateway.v1.Turn.updated_at:type_name -> google.protobuf.Timestamp + 0, // 7: gateway.v1.SnapshotResponse.sessions:type_name -> gateway.v1.Session + 48, // 8: gateway.v1.SnapshotResponse.models_by_provider:type_name -> gateway.v1.SnapshotResponse.ModelsByProviderEntry + 49, // 9: gateway.v1.SnapshotResponse.active_personas:type_name -> gateway.v1.SnapshotResponse.ActivePersonasEntry + 0, // 10: gateway.v1.GetSessionResponse.session:type_name -> gateway.v1.Session + 1, // 11: gateway.v1.RecentMessagesResponse.messages:type_name -> gateway.v1.Message + 4, // 12: gateway.v1.RecentTurnsResponse.turns:type_name -> gateway.v1.Turn + 1, // 13: gateway.v1.RouteRequest.message:type_name -> gateway.v1.Message + 1, // 14: gateway.v1.StreamRequest.message:type_name -> gateway.v1.Message + 2, // 15: gateway.v1.StreamResponse.tool:type_name -> gateway.v1.ToolCall + 3, // 16: gateway.v1.StreamResponse.media:type_name -> gateway.v1.Media + 0, // 17: gateway.v1.SaveSessionRequest.session:type_name -> gateway.v1.Session + 0, // 18: gateway.v1.GetSessionsByChannelResponse.sessions:type_name -> gateway.v1.Session + 0, // 19: gateway.v1.ListSessionsResponse.sessions:type_name -> gateway.v1.Session + 1, // 20: gateway.v1.SaveMessageRequest.message:type_name -> gateway.v1.Message + 1, // 21: gateway.v1.SaveMessagesRequest.messages:type_name -> gateway.v1.Message + 1, // 22: gateway.v1.ReplaceMessagesRequest.messages:type_name -> gateway.v1.Message + 1, // 23: gateway.v1.SearchMessagesResponse.messages:type_name -> gateway.v1.Message + 5, // 24: gateway.v1.SnapshotResponse.ModelsByProviderEntry.value:type_name -> gateway.v1.StringList + 6, // 25: gateway.v1.ChatService.Snapshot:input_type -> gateway.v1.SnapshotRequest + 8, // 26: gateway.v1.ChatService.GetSession:input_type -> gateway.v1.GetSessionRequest + 10, // 27: gateway.v1.ChatService.RecentMessages:input_type -> gateway.v1.RecentMessagesRequest + 12, // 28: gateway.v1.ChatService.RecentTurns:input_type -> gateway.v1.RecentTurnsRequest + 14, // 29: gateway.v1.ChatService.Route:input_type -> gateway.v1.RouteRequest + 16, // 30: gateway.v1.ChatService.Stream:input_type -> gateway.v1.StreamRequest + 18, // 31: gateway.v1.ChatService.Cancel:input_type -> gateway.v1.CancelRequest + 20, // 32: gateway.v1.ChatService.SetPersona:input_type -> gateway.v1.SetPersonaRequest + 22, // 33: gateway.v1.ChatService.ApplyTaskAction:input_type -> gateway.v1.ApplyTaskActionRequest + 24, // 34: gateway.v1.ChatService.ApplyOperatorTaskAction:input_type -> gateway.v1.ApplyOperatorTaskActionRequest + 27, // 35: gateway.v1.ChatService.SaveSession:input_type -> gateway.v1.SaveSessionRequest + 28, // 36: gateway.v1.ChatService.GetSessionsByChannel:input_type -> gateway.v1.GetSessionsByChannelRequest + 36, // 37: gateway.v1.ChatService.DeleteSession:input_type -> gateway.v1.DeleteSessionRequest + 37, // 38: gateway.v1.ChatService.TouchSession:input_type -> gateway.v1.TouchSessionRequest + 38, // 39: gateway.v1.ChatService.ListSessions:input_type -> gateway.v1.ListSessionsRequest + 39, // 40: gateway.v1.ChatService.SaveMessage:input_type -> gateway.v1.SaveMessageRequest + 40, // 41: gateway.v1.ChatService.DeleteRecentMessages:input_type -> gateway.v1.DeleteRecentMessagesRequest + 42, // 42: gateway.v1.ChatService.MessageCount:input_type -> gateway.v1.MessageCountRequest + 44, // 43: gateway.v1.ChatService.SaveMessages:input_type -> gateway.v1.SaveMessagesRequest + 45, // 44: gateway.v1.ChatService.ReplaceMessages:input_type -> gateway.v1.ReplaceMessagesRequest + 46, // 45: gateway.v1.ChatService.SearchMessages:input_type -> gateway.v1.SearchMessagesRequest + 7, // 46: gateway.v1.ChatService.Snapshot:output_type -> gateway.v1.SnapshotResponse + 9, // 47: gateway.v1.ChatService.GetSession:output_type -> gateway.v1.GetSessionResponse + 11, // 48: gateway.v1.ChatService.RecentMessages:output_type -> gateway.v1.RecentMessagesResponse + 13, // 49: gateway.v1.ChatService.RecentTurns:output_type -> gateway.v1.RecentTurnsResponse + 15, // 50: gateway.v1.ChatService.Route:output_type -> gateway.v1.RouteResponse + 17, // 51: gateway.v1.ChatService.Stream:output_type -> gateway.v1.StreamResponse + 19, // 52: gateway.v1.ChatService.Cancel:output_type -> gateway.v1.CancelResponse + 21, // 53: gateway.v1.ChatService.SetPersona:output_type -> gateway.v1.SetPersonaResponse + 23, // 54: gateway.v1.ChatService.ApplyTaskAction:output_type -> gateway.v1.ApplyTaskActionResponse + 25, // 55: gateway.v1.ChatService.ApplyOperatorTaskAction:output_type -> gateway.v1.ApplyOperatorTaskActionResponse + 26, // 56: gateway.v1.ChatService.SaveSession:output_type -> gateway.v1.SaveSessionResponse + 29, // 57: gateway.v1.ChatService.GetSessionsByChannel:output_type -> gateway.v1.GetSessionsByChannelResponse + 31, // 58: gateway.v1.ChatService.DeleteSession:output_type -> gateway.v1.DeleteSessionResponse + 32, // 59: gateway.v1.ChatService.TouchSession:output_type -> gateway.v1.TouchSessionResponse + 30, // 60: gateway.v1.ChatService.ListSessions:output_type -> gateway.v1.ListSessionsResponse + 33, // 61: gateway.v1.ChatService.SaveMessage:output_type -> gateway.v1.SaveMessageResponse + 41, // 62: gateway.v1.ChatService.DeleteRecentMessages:output_type -> gateway.v1.DeleteRecentMessagesResponse + 43, // 63: gateway.v1.ChatService.MessageCount:output_type -> gateway.v1.MessageCountResponse + 34, // 64: gateway.v1.ChatService.SaveMessages:output_type -> gateway.v1.SaveMessagesResponse + 35, // 65: gateway.v1.ChatService.ReplaceMessages:output_type -> gateway.v1.ReplaceMessagesResponse + 47, // 66: gateway.v1.ChatService.SearchMessages:output_type -> gateway.v1.SearchMessagesResponse + 46, // [46:67] is the sub-list for method output_type + 25, // [25:46] is the sub-list for method input_type + 25, // [25:25] is the sub-list for extension type_name + 25, // [25:25] is the sub-list for extension extendee + 0, // [0:25] is the sub-list for field type_name } func init() { file_gateway_v1_chat_proto_init() } diff --git a/internal/domain/messaging/inbound.go b/internal/domain/messaging/inbound.go index c52c4b4ea..7d32b28ba 100644 --- a/internal/domain/messaging/inbound.go +++ b/internal/domain/messaging/inbound.go @@ -46,6 +46,15 @@ type Inbound struct { // record keeps the platform. Empty means the sender did not name its // channel, and the Gateway falls back to its own name. Platform string + + // Media carries files the sender attached to this message (photos, + // documents, voice, video) with their in-process bytes already + // downloaded. Like Page and BudgetKey it is transport-only: it never + // crosses a gRPC or persistence boundary, and the stored Message + // records the attachment as a short textual note in its Text instead, + // because download URLs expire and raw media bytes must not silently + // inflate the transcript store. nil for a text-only message. + Media []MediaAttachment } // SpawnRequest is a chat-originated task creation request. Repo and diff --git a/internal/domain/messaging/media.go b/internal/domain/messaging/media.go index 8a4d8b89d..6c7f370d1 100644 --- a/internal/domain/messaging/media.go +++ b/internal/domain/messaging/media.go @@ -50,4 +50,10 @@ type MediaAttachment struct { // Raw carries platform-specific fields the common vocabulary does not // cover, so adapters never need a second attachment type. Raw map[string]any `json:"raw,omitempty"` + + // Data carries the attachment bytes in-process. An inbound path sets + // it after downloading the file; outbound senders never set it and + // read Path or URL instead. It never crosses a wire or persistence + // boundary -- it is a turn-scoped value, not a stored one. + Data []byte `json:"data,omitempty"` } diff --git a/internal/domain/messaging/message.go b/internal/domain/messaging/message.go index 5a6b47161..c318b6924 100644 --- a/internal/domain/messaging/message.go +++ b/internal/domain/messaging/message.go @@ -105,6 +105,16 @@ type Message struct { Role Role Text string + // Media carries attachment metadata for messages that arrived with a + // file (photo, document, video, audio): the platform attachment id, its + // name, type, and size, so the transcript records what was sent as + // data rather than as prose. The attachment bytes and any embedded + // Data field are deliberately NOT persisted -- they are turn-scoped -- + // so this is the metadata a later reader (search, transcript, a + // re-look capability) can act on, including re-fetching through + // FileID where the platform allows it. + Media []MediaAttachment `json:"media,omitempty"` + // ToolCall and ToolResult are populated instead of Text for their // respective Roles; a Message carries exactly one of Text, ToolCall, or // ToolResult content. diff --git a/internal/gateway/session_store_memory.go b/internal/gateway/session_store_memory.go index cfa407f06..048503c27 100644 --- a/internal/gateway/session_store_memory.go +++ b/internal/gateway/session_store_memory.go @@ -182,6 +182,7 @@ func (s *memorySessionStore) saveMessage(sessionID string, msg messaging.Message stored := messaging.Message{ ID: messaging.MessageID(id), SourceID: msg.SourceID, Sender: msg.Sender, SenderID: msg.SenderID, Role: role, Text: msg.Text, + Media: stripMediaBytes(msg.Media), } history = append(history, memoryMessage{seq: s.seq, ts: ts, msg: stored}) slices.SortStableFunc(history, func(a, b memoryMessage) int { diff --git a/internal/gateway/session_store_postgres.go b/internal/gateway/session_store_postgres.go index dde4403ef..d0ce12458 100644 --- a/internal/gateway/session_store_postgres.go +++ b/internal/gateway/session_store_postgres.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "encoding/json" "errors" "fmt" "math" @@ -238,11 +239,16 @@ func saveMessagePG(ctx context.Context, q *postgresdb.Queries, sessionID string, } } at := stamp(msg) + mediaJSON, err := marshalMediaMeta(msg.Media) + if err != nil { + return "", err + } if clamp { err := q.InsertMessageClamped(ctx, postgresdb.InsertMessageClampedParams{ MessageID: id, SessionID: sessionID, SourceID: msg.SourceID, Sender: msg.Sender, SenderID: msg.SenderID, Role: role, Text: msg.Text, - Ts: at.UnixMilli(), + Media: mediaJSON, + Ts: at.UnixMilli(), }) if err != nil { return "", fmt.Errorf("sessionstore: save message: %w", err) @@ -263,7 +269,8 @@ func saveMessagePG(ctx context.Context, q *postgresdb.Queries, sessionID string, err = q.InsertMessageAt(ctx, postgresdb.InsertMessageAtParams{ MessageID: id, SessionID: sessionID, SourceID: msg.SourceID, Sender: msg.Sender, SenderID: msg.SenderID, Role: role, Text: msg.Text, - Ts: at.UnixMilli(), + Media: mediaJSON, + Ts: at.UnixMilli(), }) if err != nil { return "", fmt.Errorf("sessionstore: save message: %w", err) @@ -393,7 +400,7 @@ func (s *postgresSessionStore) MessageCount(ctx context.Context, sessionID strin func recentMessagesFromRows(rows []postgresdb.RecentMessagesRow) []messaging.Message { out := make([]messaging.Message, 0, len(rows)) for _, row := range rows { - out = append(out, messaging.Message{ + out = append(out, rowMessage(messaging.Message{ ID: messaging.MessageID(row.MessageID), SourceID: row.SourceID, Sender: row.Sender, @@ -404,11 +411,66 @@ func recentMessagesFromRows(rows []postgresdb.RecentMessagesRow) []messaging.Mes ConversationID: messaging.ConversationID{ ChannelID: row.ChannelID, ThreadID: row.ThreadID, }, - }) + }, row.Media)) } return out } +// stripMediaBytes returns the attachment list with in-process bytes +// removed -- the shared persist-boundary rule both stores apply. Data is a +// turn-scoped in-process value: a transcript that quietly retained +// megabytes of base64 would corrupt the context-budget model the message +// text assumes. Metadata survives; bytes do not. +func stripMediaBytes(media []messaging.MediaAttachment) []messaging.MediaAttachment { + if len(media) == 0 { + return nil + } + stripped := make([]messaging.MediaAttachment, len(media)) + for i, att := range media { + stripped[i] = att + stripped[i].Data = nil + } + return stripped +} + +// marshalMediaMeta serialises a message's attachment metadata for the +// database JSON column. +func marshalMediaMeta(media []messaging.MediaAttachment) ([]byte, error) { + if len(media) == 0 { + return []byte("[]"), nil + } + encoded, err := json.Marshal(stripMediaBytes(media)) + if err != nil { + return nil, fmt.Errorf("sessionstore: marshal message media: %w", err) + } + return encoded, nil +} + +// unmarshalMediaMeta decodes the media JSON column a message row carries. +func unmarshalMediaMeta(raw string) ([]messaging.MediaAttachment, error) { + if raw == "" || raw == "[]" { + return nil, nil + } + var media []messaging.MediaAttachment + if err := json.Unmarshal([]byte(raw), &media); err != nil { + return nil, fmt.Errorf("sessionstore: unmarshal message media: %w", err) + } + return media, nil +} + +// rowMessage attaches a message row's decoded media to its base record; +// a row's media column can only fail to decode through corruption, which +// is surfaced rather than silently dropping the attachments. +func rowMessage(base messaging.Message, mediaRaw string) messaging.Message { + media, err := unmarshalMediaMeta(mediaRaw) + if err != nil { + base.Text += " [unreadable media metadata]" + return base + } + base.Media = media + return base +} + // SearchMessages runs a tsvector search over the session's entire message // history, matching message text and sender only. The generated search column // indexes exactly (sender, text), so ts, source_id and the other metadata are @@ -461,7 +523,7 @@ func (s *postgresSessionStore) SearchMessages(ctx context.Context, sessionID str msgs := make([]messaging.Message, 0, len(rows)) for _, row := range rows { - msgs = append(msgs, messaging.Message{ + msgs = append(msgs, rowMessage(messaging.Message{ ID: messaging.MessageID(row.MessageID), SourceID: row.SourceID, Sender: row.Sender, @@ -472,7 +534,7 @@ func (s *postgresSessionStore) SearchMessages(ctx context.Context, sessionID str ConversationID: messaging.ConversationID{ ChannelID: row.ChannelID, ThreadID: row.ThreadID, }, - }) + }, row.Media)) } next := offset + len(msgs) diff --git a/internal/gateway/sessionstore_conformance_test.go b/internal/gateway/sessionstore_conformance_test.go index bc3227eec..cf60bc783 100644 --- a/internal/gateway/sessionstore_conformance_test.go +++ b/internal/gateway/sessionstore_conformance_test.go @@ -75,6 +75,7 @@ func runSessionStoreSuite(t *testing.T, newStore func(t *testing.T) SessionStore t.Run("ReplyCannotPrecedeItsPrompt", func(t *testing.T) { testReplyCannotPrecedeItsPrompt(t, newStore) }) t.Run("UpstreamRedeliveryKeepsOriginalTime", func(t *testing.T) { testUpstreamRedeliveryKeepsOriginalTime(t, newStore) }) t.Run("SaveMessageStampsMissingTimestamp", func(t *testing.T) { testSaveMessageStampsMissingTimestamp(t, newStore) }) + t.Run("AttachmentMetadataPersistsWithoutBytes", func(t *testing.T) { testAttachmentMetadataPersistsWithoutBytes(t, newStore) }) t.Run("CanonicalMessageIDIsStable", func(t *testing.T) { testCanonicalMessageIDIsStable(t, newStore) }) t.Run("SourceIDRoundTrips", func(t *testing.T) { testSourceIDRoundTrips(t, newStore) }) t.Run("SourceIDIsNotSearchable", func(t *testing.T) { testSourceIDIsNotSearchable(t, newStore) }) @@ -535,6 +536,49 @@ func testUpstreamRedeliveryKeepsOriginalTime(t *testing.T, newStore func(t *test } } +// testAttachmentMetadataPersistsWithoutBytes pins what kind of media +// record the session log keeps: the attachment's metadata (what it is, +// where it lives on the platform, how big it is) persists as data, while +// the in-process bytes -- turn-scoped by design -- must never enter the +// store. Losing the bytes across turns is deliberate; losing the fact that +// a photo was sent is not. +func testAttachmentMetadataPersistsWithoutBytes(t *testing.T, newStore func(t *testing.T) SessionStore) { + ctx := context.Background() + s := newStore(t) + t.Cleanup(func() { _ = s.Close() }) + + size, width, duration := int64(150000), 1280, 12 + media := []messaging.MediaAttachment{{ + Type: "image", FileID: "tg-file-1", MIMEType: "image/jpeg", + FileName: "cat.jpg", FileSize: &size, Width: &width, Duration: &duration, + Data: []byte("in-process bytes"), + }} + if err := s.SaveMessage(ctx, "sess", messaging.Message{ + Role: messaging.RoleUser, Text: "[photo]\nlook at this", Media: media, + }); err != nil { + t.Fatalf("SaveMessage: %v", err) + } + got, err := s.RecentMessages(ctx, "sess", 1) + if err != nil { + t.Fatalf("RecentMessages: %v", err) + } + if len(got) != 1 || len(got[0].Media) != 1 { + t.Fatalf("stored message = %#v, want one message with one attachment", got) + } + att := got[0].Media[0] + if att.Type != "image" || att.FileID != "tg-file-1" || att.MIMEType != "image/jpeg" || + att.FileName != "cat.jpg" || att.FileSize == nil || *att.FileSize != size || + att.Width == nil || *att.Width != width || att.Duration == nil || *att.Duration != duration { + t.Errorf("stored attachment = %#v, want the full metadata", att) + } + if len(att.Data) != 0 { + t.Errorf("stored message carries %d attachment bytes, want none", len(att.Data)) + } + if got[0].Text != "[photo]\nlook at this" { + t.Errorf("Text = %q, want the placeholder note preserved", got[0].Text) + } +} + // testSaveMessageStampsMissingTimestamp pins that a caller omitting At still // gets a usable ordering key rather than the zero time. This is the one // legitimate use of time.Now bounds: the store itself stamps "now" when At diff --git a/internal/gateway/turn.go b/internal/gateway/turn.go index e44e10029..f6f5c7221 100644 --- a/internal/gateway/turn.go +++ b/internal/gateway/turn.go @@ -45,7 +45,12 @@ type PreparedTurnModel interface { // TurnModelRequest is the provider-neutral request assembled by the gateway. // Messages include the system prompt as their first entry. type TurnModelRequest struct { - Messages []CompressedMessage + Messages []CompressedMessage + // Media carries the media attachments of this turn's inbound message. + // They belong to the final user message and are never part of stored + // history, so the model seam must merge them into that message when + // building its provider request. Empty for a text-only turn. + Media []messaging.MediaAttachment MaxOutputTokens int } @@ -294,13 +299,15 @@ func (r *TurnRunner) recordInboundMessage( } // preparedTurn is the generation-ready state built from a turn's history: -// the prepared model, its resolved details, and the compressed message -// view (system prompt already prepended). +// the prepared model, its resolved details, the compressed message view +// (system prompt already prepended), and the current turn's media which +// lives outside stored history and so must ride the prepared turn itself. type preparedTurn struct { prepared PreparedTurnModel modelName string modelDetails ModelDetails view CompressedView + media []messaging.MediaAttachment } // prepareTurn builds the tools, system prompt, and compressed history view @@ -383,7 +390,7 @@ func (r *TurnRunner) prepareTurn(ctx context.Context, sessionID string, in Inbou view.Messages = append( []CompressedMessage{{Role: "system", Content: systemPrompt}}, view.Messages..., ) - return preparedTurn{prepared: prepared, modelName: modelName, modelDetails: modelDetails, view: view}, nil + return preparedTurn{prepared: prepared, modelName: modelName, modelDetails: modelDetails, view: view, media: in.Media}, nil } // compressSessionAtBudget compresses the session's stored history when it has @@ -490,6 +497,7 @@ func (r *TurnRunner) generateAndComplete( recorder := &toolCallRecorder{next: stream} text, err := prep.prepared.Generate(ctx, TurnModelRequest{ Messages: prep.view.Messages, + Media: prep.media, MaxOutputTokens: prep.modelDetails.MaxOutputTokens, }, recorder) if err != nil { diff --git a/internal/gateway/turn_media_test.go b/internal/gateway/turn_media_test.go new file mode 100644 index 000000000..8e04d2a08 --- /dev/null +++ b/internal/gateway/turn_media_test.go @@ -0,0 +1,56 @@ +package gateway + +import ( + "context" + "log/slog" + "reflect" + "testing" + + "github.com/samcharles93/archie-core/internal/domain/messaging" +) + +// TestTurnRunnerGivesInboundMediaToTheModelRequest pins the handoff point +// between the adapter and the model seam: media a channel attached to an +// inbound message must reach the model request, or the channel's work is +// lost and the turn answers on caption text alone. +func TestTurnRunnerGivesInboundMediaToTheModelRequest(t *testing.T) { + store := NewSessionStoreMemory() + router := NewRouter(nil, nil, "telegram") + router.Identity = "archie" + router.InitSessions(store) + + prepared := &turnTestPreparedModel{reply: "answer"} + model := &turnTestModel{prepared: prepared} + runner := NewTurnRunner(TurnRunnerConfig{ + Router: router, + Sessions: store, + Models: &fakeModelManager{models: []string{"test/model"}, activeModel: "test/model"}, + Personas: NewPersonaRegistry(DefaultPersonas()), + Model: model, + BotUser: "archie", + Channel: "telegram", + Operator: "Sam", + Log: slog.New(slog.DiscardHandler), + }) + + media := []messaging.MediaAttachment{{Type: "image", MIMEType: "image/png", Data: []byte("png-bytes")}} + _, err := runner.Run(context.Background(), Inbound{ + Message: messaging.Message{ + SourceID: "src-media", + ConversationID: messaging.ConversationID{ChannelID: "chat-media"}, + Sender: "user", + Role: messaging.RoleUser, + Text: "look at this", + }, + Media: media, + }, DeltaFunc(func(string) {})) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if len(prepared.lastRequest.Media) != 1 { + t.Fatalf("model request media = %#v, want the one inbound attachment", prepared.lastRequest.Media) + } + if !reflect.DeepEqual(prepared.lastRequest.Media, media) { + t.Errorf("model request media = %#v, want %#v", prepared.lastRequest.Media, media) + } +} diff --git a/internal/infrastructure/gatewayrpc/values.go b/internal/infrastructure/gatewayrpc/values.go index b519e5b69..042a65e93 100644 --- a/internal/infrastructure/gatewayrpc/values.go +++ b/internal/infrastructure/gatewayrpc/values.go @@ -35,6 +35,7 @@ func storedProto(m messaging.Message) *pb.Message { From: m.Sender, Text: m.Text, At: timestamp(m.At), + Media: attachmentProtoList(m.Media), } } @@ -51,6 +52,7 @@ func storedValue(v *pb.Message) messaging.Message { Sender: v.From, Text: v.Text, At: timeValue(v.At), + Media: attachmentValueList(v.GetMedia()), } } @@ -62,6 +64,7 @@ func inboundProto(v messaging.Inbound) *pb.Message { m.Page = v.Page m.BudgetKey = v.BudgetKey m.Platform = v.Platform + m.Media = attachmentProtoList(v.Media) return m } @@ -75,7 +78,7 @@ func inboundValue(v *pb.Message) messaging.Inbound { } msg := storedValue(v) msg.Role = messaging.RoleUser - return messaging.Inbound{Message: msg, Page: v.Page, BudgetKey: v.BudgetKey, Platform: v.Platform} + return messaging.Inbound{Message: msg, Page: v.Page, BudgetKey: v.BudgetKey, Platform: v.Platform, Media: attachmentValueList(v.GetMedia())} } func toolProto(v messaging.ToolCallEvent) *pb.ToolCall { @@ -172,16 +175,48 @@ func intPointer(v *int64) *int { return &n } +// attachmentProtoList renders a message's attachments in wire shape. Data +// is deliberately unmapped: it is a turn-scoped in-process value and must +// never cross the wire, matching the persisted record which strips it too. +// An attachment with no fields beyond Data would produce an empty wire +// message, which is still the honest encoding of "metadata was stripped", +// and never happens for a real attachment. +func attachmentProtoList(media []messaging.MediaAttachment) []*pb.Media { + if len(media) == 0 { + return nil + } + return mapValues(media, attachmentProto) +} + +func attachmentValueList(list []*pb.Media) []messaging.MediaAttachment { + if len(list) == 0 { + return nil + } + return mapValues(list, attachmentValue) +} + +func attachmentProto(a messaging.MediaAttachment) *pb.Media { + return &pb.Media{Type: a.Type, FileId: a.FileID, Url: a.URL, Path: a.Path, MimeType: a.MIMEType, FileName: a.FileName, FileSize: a.FileSize, Width: int64Pointer(a.Width), Height: int64Pointer(a.Height), Duration: int64Pointer(a.Duration)} +} + +func attachmentValue(v *pb.Media) messaging.MediaAttachment { + if v == nil { + return messaging.MediaAttachment{} + } + return messaging.MediaAttachment{Type: v.Type, FileID: v.FileId, URL: v.Url, Path: v.Path, MIMEType: v.MimeType, FileName: v.FileName, FileSize: v.FileSize, Width: intPointer(v.Width), Height: intPointer(v.Height), Duration: intPointer(v.Duration)} +} + func mediaProto(v messaging.MediaEvent) *pb.Media { - a := v.Attachment - return &pb.Media{ToolName: v.ToolName, Type: a.Type, FileId: a.FileID, Url: a.URL, Path: a.Path, MimeType: a.MIMEType, FileName: a.FileName, FileSize: a.FileSize, Width: int64Pointer(a.Width), Height: int64Pointer(a.Height), Duration: int64Pointer(a.Duration)} + pm := attachmentProto(v.Attachment) + pm.ToolName = v.ToolName + return pm } func mediaValue(v *pb.Media) messaging.MediaEvent { if v == nil { return messaging.MediaEvent{} } - return messaging.MediaEvent{ToolName: v.ToolName, Attachment: messaging.MediaAttachment{Type: v.Type, FileID: v.FileId, URL: v.Url, Path: v.Path, MIMEType: v.MimeType, FileName: v.FileName, FileSize: v.FileSize, Width: intPointer(v.Width), Height: intPointer(v.Height), Duration: intPointer(v.Duration)}} + return messaging.MediaEvent{ToolName: v.ToolName, Attachment: attachmentValue(v)} } func eventProto(v messaging.ChatEvent) *pb.StreamResponse { diff --git a/internal/infrastructure/gatewayrpc/values_media_test.go b/internal/infrastructure/gatewayrpc/values_media_test.go new file mode 100644 index 000000000..b51a10321 --- /dev/null +++ b/internal/infrastructure/gatewayrpc/values_media_test.go @@ -0,0 +1,62 @@ +package gatewayrpc + +import ( + "testing" + + "github.com/samcharles93/archie-core/internal/domain/messaging" +) + +// TestStoredMediaRoundTrip: a message's attachment metadata must survive +// the wire both ways, while the in-process bytes -- turn-scoped by +// contract -- must never be marshalled onto it. +func TestStoredMediaRoundTrip(t *testing.T) { + size, width := int64(2048), 640 + in := messaging.Message{ + ID: messaging.MessageID("m1"), + Role: messaging.RoleUser, Text: "[document: notes.txt]", + Media: []messaging.MediaAttachment{{ + Type: "document", FileID: "tg-f1", URL: "https://example.invalid/notes", + Path: "/tmp/never-here", MIMEType: "text/plain", FileName: "notes.txt", + FileSize: &size, Width: &width, Data: []byte("file bytes"), + }}, + } + out := storedValue(storedProto(in)) + if len(out.Media) != 1 { + t.Fatalf("round-tripped media = %#v, want one attachment", out.Media) + } + att := out.Media[0] + if att.Type != "document" || att.FileName != "notes.txt" || att.MIMEType != "text/plain" { + t.Errorf("attachment = %#v, want its metadata preserved", att) + } + if att.FileID != "tg-f1" { + t.Errorf("FileID = %q, want tg-f1", att.FileID) + } + if att.FileSize == nil || *att.FileSize != size || att.Width == nil || *att.Width != width { + t.Errorf("numeric metadata = %#v, want size and width preserved", att) + } + if att.URL != "https://example.invalid/notes" { + t.Errorf("URL = %q, want preserved", att.URL) + } + if len(att.Data) != 0 { + t.Errorf("attachment bytes crossed the wire, want them stripped") + } +} + +// TestInboundMediaRoundTrip mirrors the stored direction for the inbound +// transport context: a frontend that names attachments on Route gets them +// back on the daemon side, minus the bytes which it must carry itself. +func TestInboundMediaRoundTrip(t *testing.T) { + in := messaging.Inbound{ + Message: messaging.Message{Text: "hi", Sender: "sam"}, + Media: []messaging.MediaAttachment{ + {Type: "image", FileID: "f1", MIMEType: "image/jpeg", Data: []byte("jpeg")}, + }, + } + out := inboundValue(inboundProto(in)) + if len(out.Media) != 1 || out.Media[0].FileID != "f1" || out.Media[0].Type != "image" { + t.Fatalf("round-tripped inbound media = %#v, want the sender attachment", out.Media) + } + if len(out.Media[0].Data) != 0 { + t.Errorf("inbound attachment bytes crossed the wire, want them stripped") + } +} diff --git a/internal/infrastructure/postgres/migrations/0020_message_media.sql b/internal/infrastructure/postgres/migrations/0020_message_media.sql new file mode 100644 index 000000000..042a37a5e --- /dev/null +++ b/internal/infrastructure/postgres/migrations/0020_message_media.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- Attachment metadata on stored chat messages (messaging.Message.Media): +-- type, platform attachment id, name, mime, size. The attachment bytes are +-- deliberately not kept -- they are turn-scoped -- so this is a JSON +-- metadata list, not a blob column. +ALTER TABLE messages ADD COLUMN media jsonb NOT NULL DEFAULT '[]'; + +-- +goose Down +ALTER TABLE messages DROP COLUMN media; \ No newline at end of file diff --git a/internal/infrastructure/postgres/postgresdb/gateway.sql.go b/internal/infrastructure/postgres/postgresdb/gateway.sql.go index 40f350942..532c7a6af 100644 --- a/internal/infrastructure/postgres/postgresdb/gateway.sql.go +++ b/internal/infrastructure/postgres/postgresdb/gateway.sql.go @@ -110,8 +110,8 @@ func (q *Queries) FinalizeTurn(ctx context.Context, arg FinalizeTurnParams) (int } const insertMessageAt = `-- name: InsertMessageAt :exec -INSERT INTO messages (message_id, session_id, source_id, sender, sender_id, role, text, ts) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +INSERT INTO messages (message_id, session_id, source_id, sender, sender_id, role, text, media, ts) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (session_id, message_id) DO NOTHING ` @@ -123,6 +123,7 @@ type InsertMessageAtParams struct { SenderID string Role string Text string + Media []byte Ts int64 } @@ -136,17 +137,18 @@ func (q *Queries) InsertMessageAt(ctx context.Context, arg InsertMessageAtParams arg.SenderID, arg.Role, arg.Text, + arg.Media, arg.Ts, ) return err } const insertMessageClamped = `-- name: InsertMessageClamped :exec -INSERT INTO messages (message_id, session_id, source_id, sender, sender_id, role, text, ts) -VALUES ($1, $2, $3, $4, $5, $6, $7, ( +INSERT INTO messages (message_id, session_id, source_id, sender, sender_id, role, text, media, ts) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, ( SELECT CASE - WHEN MAX(m.ts) IS NOT NULL AND MAX(m.ts) >= $8 THEN MAX(m.ts) + 1 - ELSE $8 + WHEN MAX(m.ts) IS NOT NULL AND MAX(m.ts) >= $9 THEN MAX(m.ts) + 1 + ELSE $9 END FROM messages m WHERE m.session_id = $2 )) @@ -161,6 +163,7 @@ type InsertMessageClampedParams struct { SenderID string Role string Text string + Media []byte Ts int64 } @@ -177,6 +180,7 @@ func (q *Queries) InsertMessageClamped(ctx context.Context, arg InsertMessageCla arg.SenderID, arg.Role, arg.Text, + arg.Media, arg.Ts, ) return err @@ -365,10 +369,10 @@ func (q *Queries) NextReplyAfterTs(ctx context.Context, arg NextReplyAfterTsPara } const recentMessages = `-- name: RecentMessages :many -SELECT m.message_id, m.source_id, m.sender, m.sender_id, m.role, m.text, m.ts, +SELECT m.message_id, m.source_id, m.sender, m.sender_id, m.role, m.text, m.ts, m.media::text AS media, COALESCE(s.channel_id, '') AS channel_id, COALESCE(s.thread_id, '') AS thread_id FROM ( - SELECT msg.message_id, msg.source_id, msg.sender, msg.sender_id, msg.role, msg.text, msg.ts, msg.session_id + SELECT msg.message_id, msg.source_id, msg.sender, msg.sender_id, msg.role, msg.text, msg.ts, msg.media::text AS media, msg.session_id FROM messages msg WHERE msg.session_id = $1 ORDER BY msg.ts DESC LIMIT $2 ) m LEFT JOIN sessions s ON s.session_id = m.session_id @@ -388,6 +392,7 @@ type RecentMessagesRow struct { Role string Text string Ts int64 + Media string ChannelID string ThreadID string } @@ -409,6 +414,7 @@ func (q *Queries) RecentMessages(ctx context.Context, arg RecentMessagesParams) &i.Role, &i.Text, &i.Ts, + &i.Media, &i.ChannelID, &i.ThreadID, ); err != nil { @@ -633,7 +639,7 @@ func (q *Queries) SearchMessagesCount(ctx context.Context, arg SearchMessagesCou } const searchMessagesPage = `-- name: SearchMessagesPage :many -SELECT m.message_id, m.source_id, m.sender, m.sender_id, m.role, m.text, m.ts, +SELECT m.message_id, m.source_id, m.sender, m.sender_id, m.role, m.text, m.ts, m.media::text AS media, COALESCE(s.channel_id, '') AS channel_id, COALESCE(s.thread_id, '') AS thread_id FROM messages m LEFT JOIN sessions s ON s.session_id = m.session_id @@ -657,6 +663,7 @@ type SearchMessagesPageRow struct { Role string Text string Ts int64 + Media string ChannelID string ThreadID string } @@ -683,6 +690,7 @@ func (q *Queries) SearchMessagesPage(ctx context.Context, arg SearchMessagesPage &i.Role, &i.Text, &i.Ts, + &i.Media, &i.ChannelID, &i.ThreadID, ); err != nil { diff --git a/internal/infrastructure/postgres/postgresdb/models.go b/internal/infrastructure/postgres/postgresdb/models.go index 0fd052b1a..0295e8643 100644 --- a/internal/infrastructure/postgres/postgresdb/models.go +++ b/internal/infrastructure/postgres/postgresdb/models.go @@ -240,6 +240,7 @@ type Message struct { Text string Ts int64 Search interface{} + Media []byte } type Org struct { diff --git a/internal/infrastructure/postgres/queries/gateway.sql b/internal/infrastructure/postgres/queries/gateway.sql index c1dc9a334..0d280eac1 100644 --- a/internal/infrastructure/postgres/queries/gateway.sql +++ b/internal/infrastructure/postgres/queries/gateway.sql @@ -61,30 +61,30 @@ SELECT pg_advisory_xact_lock(hashtextextended('messages:' || $1::text, 0)); -- SQL, not in Go: the CASE assigns MAX(ts)+1 when the caller's timestamp is -- not ahead of the session's newest, else the caller's timestamp. A Go-side -- read-then-write would race two concurrent appends to the same session. -INSERT INTO messages (message_id, session_id, source_id, sender, sender_id, role, text, ts) -VALUES ($1, $2, $3, $4, $5, $6, $7, ( +INSERT INTO messages (message_id, session_id, source_id, sender, sender_id, role, text, media, ts) +VALUES (@message_id, @session_id, @source_id, @sender, @sender_id, @role, @text, @media, ( SELECT CASE - WHEN MAX(m.ts) IS NOT NULL AND MAX(m.ts) >= $8 THEN MAX(m.ts) + 1 - ELSE $8 + WHEN MAX(m.ts) IS NOT NULL AND MAX(m.ts) >= @ts THEN MAX(m.ts) + 1 + ELSE @ts END - FROM messages m WHERE m.session_id = $2 + FROM messages m WHERE m.session_id = @session_id )) ON CONFLICT (session_id, message_id) DO NOTHING; -- name: InsertMessageAt :exec -- Rewriting history places records deliberately, so no clamp applies. -INSERT INTO messages (message_id, session_id, source_id, sender, sender_id, role, text, ts) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +INSERT INTO messages (message_id, session_id, source_id, sender, sender_id, role, text, media, ts) +VALUES (@message_id, @session_id, @source_id, @sender, @sender_id, @role, @text, @media, @ts) ON CONFLICT (session_id, message_id) DO NOTHING; -- name: MessageID :one SELECT message_id FROM messages WHERE session_id = $1 AND message_id = $2; -- name: RecentMessages :many -SELECT m.message_id, m.source_id, m.sender, m.sender_id, m.role, m.text, m.ts, +SELECT m.message_id, m.source_id, m.sender, m.sender_id, m.role, m.text, m.ts, m.media::text AS media, COALESCE(s.channel_id, '') AS channel_id, COALESCE(s.thread_id, '') AS thread_id FROM ( - SELECT msg.message_id, msg.source_id, msg.sender, msg.sender_id, msg.role, msg.text, msg.ts, msg.session_id + SELECT msg.message_id, msg.source_id, msg.sender, msg.sender_id, msg.role, msg.text, msg.ts, msg.media::text AS media, msg.session_id FROM messages msg WHERE msg.session_id = $1 ORDER BY msg.ts DESC LIMIT $2 ) m LEFT JOIN sessions s ON s.session_id = m.session_id @@ -106,7 +106,7 @@ SELECT COUNT(*) FROM messages WHERE session_id = $1 AND search @@ plainto_tsquery('simple', $2); -- name: SearchMessagesPage :many -SELECT m.message_id, m.source_id, m.sender, m.sender_id, m.role, m.text, m.ts, +SELECT m.message_id, m.source_id, m.sender, m.sender_id, m.role, m.text, m.ts, m.media::text AS media, COALESCE(s.channel_id, '') AS channel_id, COALESCE(s.thread_id, '') AS thread_id FROM messages m LEFT JOIN sessions s ON s.session_id = m.session_id diff --git a/proto/gateway/v1/chat.proto b/proto/gateway/v1/chat.proto index 1db6c92f3..da07b758f 100644 --- a/proto/gateway/v1/chat.proto +++ b/proto/gateway/v1/chat.proto @@ -63,6 +63,11 @@ message Message { // Source.Platform -- the first component of the session natural key -- a // constant. Transport-only, never persisted: the session record keeps it. string platform = 10; + // media carries attachment metadata for a message that arrived with a + // file. The attachment's bytes never cross this wire: a stored Media + // with tool_name empty is a sender attachment, and the tool_name on + // tool-produced events stays a stream-only field. + repeated Media media = 11; } message ToolCall { string id = 1;