Merge pull request elizaOS#1220 from ai16z/tcm-telegram-image

fix: Allow the bot to post messages with images generated by the imageGenerationPlugin on Telegram.
AbdelStark · Dec 19, 2024 · 392efc6 · 392efc6
2 parents d3d6198 + cd5fc2f
commit 392efc6
Show file tree

Hide file tree

Showing 3 changed files with 109 additions and 67 deletions.
diff --git a/packages/client-telegram/src/messageManager.ts b/packages/client-telegram/src/messageManager.ts
@@ -12,6 +12,7 @@ import {
     ModelClass,
     State,
     UUID,
+    Media,
 } from "@ai16z/eliza";
 import { stringToUuid } from "@ai16z/eliza";
 
@@ -26,6 +27,8 @@ import {
     TEAM_COORDINATION
 } from "./constants";
 
+import fs from "fs";
+
 const MAX_MESSAGE_LENGTH = 4096; // Telegram's max message length
 
 const telegramShouldRespondTemplate =
@@ -580,29 +583,77 @@ export class MessageManager {
     // Send long messages in chunks
     private async sendMessageInChunks(
         ctx: Context,
-        content: string,
+        content: Content,
         replyToMessageId?: number
     ): Promise<Message.TextMessage[]> {
-        const chunks = this.splitMessage(content);
-        const sentMessages: Message.TextMessage[] = [];
-
-        for (let i = 0; i < chunks.length; i++) {
-            const chunk = chunks[i];
-            const sentMessage = (await ctx.telegram.sendMessage(
-                ctx.chat.id,
-                chunk,
-                {
-                    reply_parameters:
-                        i === 0 && replyToMessageId
-                            ? { message_id: replyToMessageId }
-                            : undefined,
+        if (content.attachments && content.attachments.length > 0) {
+            content.attachments.map(async (attachment: Media) => {
+                if (attachment.contentType.startsWith("image")) {
+                    this.sendImage(ctx, attachment.url, attachment.description);
                 }
-            )) as Message.TextMessage;
+            });
+        } else {
+            const chunks = this.splitMessage(content.text);
+            const sentMessages: Message.TextMessage[] = [];
+
+            for (let i = 0; i < chunks.length; i++) {
+                const chunk = chunks[i];
+                const sentMessage = (await ctx.telegram.sendMessage(
+                    ctx.chat.id,
+                    chunk,
+                    {
+                        reply_parameters:
+                            i === 0 && replyToMessageId
+                                ? { message_id: replyToMessageId }
+                                : undefined,
+                    }
+                )) as Message.TextMessage;
+
+                sentMessages.push(sentMessage);
+            }
 
-            sentMessages.push(sentMessage);
+            return sentMessages;
         }
+    }
 
-        return sentMessages;
+    private async sendImage(
+        ctx: Context,
+        imagePath: string,
+        caption?: string
+    ): Promise<void> {
+        try {
+            if (/^(http|https):\/\//.test(imagePath)) {
+                // Handle HTTP URLs
+                await ctx.telegram.sendPhoto(
+                    ctx.chat.id,
+                    imagePath,
+                    {
+                        caption,
+                    }
+                );
+            } else {
+                // Handle local file paths 
+                if (!fs.existsSync(imagePath)) {
+                    throw new Error(`File not found: ${imagePath}`);
+                }
+
+                const fileStream = fs.createReadStream(imagePath);
+
+                await ctx.telegram.sendPhoto(
+                    ctx.chat.id,
+                    {
+                        source: fileStream,
+                    },
+                    {
+                        caption,
+                    }
+                );
+            }
+
+            elizaLogger.info(`Image sent successfully: ${imagePath}`);
+        } catch (error) {
+            elizaLogger.error("Error sending image:", error);
+        }
     }
 
     // Split message into smaller parts
@@ -906,46 +957,47 @@ export class MessageManager {
                 const callback: HandlerCallback = async (content: Content) => {
                     const sentMessages = await this.sendMessageInChunks(
                         ctx,
-                        content.text,
+                        content,
                         message.message_id
                     );
-
-                    const memories: Memory[] = [];
-
-                    // Create memories for each sent message
-                    for (let i = 0; i < sentMessages.length; i++) {
-                        const sentMessage = sentMessages[i];
-                        const isLastMessage = i === sentMessages.length - 1;
-
-                        const memory: Memory = {
-                            id: stringToUuid(
-                                sentMessage.message_id.toString() +
-                                    "-" +
-                                    this.runtime.agentId
-                            ),
-                            agentId,
-                            userId: agentId,
-                            roomId,
-                            content: {
-                                ...content,
-                                text: sentMessage.text,
-                                inReplyTo: messageId,
-                            },
-                            createdAt: sentMessage.date * 1000,
-                            embedding: getEmbeddingZeroVector(),
-                        };
-
-                        // Set action to CONTINUE for all messages except the last one
-                        // For the last message, use the original action from the response content
-                        memory.content.action = !isLastMessage
-                            ? "CONTINUE"
-                            : content.action;
-
-                        await this.runtime.messageManager.createMemory(memory);
-                        memories.push(memory);
+                    if (sentMessages) {
+                        const memories: Memory[] = [];
+
+                        // Create memories for each sent message
+                        for (let i = 0; i < sentMessages.length; i++) {
+                            const sentMessage = sentMessages[i];
+                            const isLastMessage = i === sentMessages.length - 1;
+
+                            const memory: Memory = {
+                                id: stringToUuid(
+                                    sentMessage.message_id.toString() +
+                                        "-" +
+                                        this.runtime.agentId
+                                ),
+                                agentId,
+                                userId: agentId,
+                                roomId,
+                                content: {
+                                    ...content,
+                                    text: sentMessage.text,
+                                    inReplyTo: messageId,
+                                },
+                                createdAt: sentMessage.date * 1000,
+                                embedding: getEmbeddingZeroVector(),
+                            };
+
+                            // Set action to CONTINUE for all messages except the last one
+                            // For the last message, use the original action from the response content
+                            memory.content.action = !isLastMessage
+                                ? "CONTINUE"
+                                : content.action;
+
+                            await this.runtime.messageManager.createMemory(memory);
+                            memories.push(memory);
+                        }
+
+                        return memories;
                     }
-
-                    return memories;
                 };
 
                 // Execute callback to send messages and log memories

diff --git a/packages/client-twitter/src/utils.ts b/packages/client-twitter/src/utils.ts
@@ -165,16 +165,6 @@ export async function buildConversationThread(
     return thread;
 }
 
-export function getMediaType(attachment: Media) {
-    if (attachment.contentType?.startsWith("video")) {
-        return "video";
-    } else if (attachment.contentType?.startsWith("image")) {
-        return "image";
-    } else {
-        throw new Error(`Unsupported media type`);
-    }
-}
-
 export async function sendTweet(
     client: ClientBase,
     content: Content,
@@ -207,14 +197,14 @@ export async function sendTweet(
                         const mediaBuffer = Buffer.from(
                             await response.arrayBuffer()
                         );
-                        const mediaType = getMediaType(attachment);
+                        const mediaType = attachment.contentType;
                         return { data: mediaBuffer, mediaType };
                     } else if (fs.existsSync(attachment.url)) {
                         // Handle local file paths
                         const mediaBuffer = await fs.promises.readFile(
                             path.resolve(attachment.url)
                         );
-                        const mediaType = getMediaType(attachment);
+                        const mediaType = attachment.contentType;
                         return { data: mediaBuffer, mediaType };
                     } else {
                         throw new Error(

diff --git a/packages/plugin-image-generation/src/index.ts b/packages/plugin-image-generation/src/index.ts
@@ -207,7 +207,7 @@ const imageGeneration: Action = {
                                 source: "imageGeneration",
                                 description: "...", //caption.title,
                                 text: "...", //caption.description,
-                                contentType: "image",
+                                contentType: "image/png",
                             },
                         ],
                     },