feat: Modifying schema to support multi modal inputs.

memgpt/agent_store/db.py

@@ -155,7 +155,7 @@ class MessageModel(Base):
 
             # openai info
             role = Column(String, nullable=False)
-            text = Column(String)  # optional: can be null if function call
+            content = Column(JSON)  # optional: multi-modal input. can be null be function call.
             model = Column(String)  # optional: can be null if LLM backend doesn't require specifying
             name = Column(String)  # optional: multi-agent only
 
@@ -192,6 +192,7 @@ def to_record(self):
                     role=self.role,
                     name=self.name,
                     text=self.text,
+                    mm_content=self.mm_content,
                     model=self.model,
                     # tool_calls=[ToolCall(id=tool_call["id"], function=ToolCallFunction(**tool_call["function"])) for tool_call in self.tool_calls] if self.tool_calls else None,
                     tool_calls=self.tool_calls,

memgpt/schemas/message.py

@@ -11,7 +11,7 @@
 from memgpt.schemas.enums import MessageRole
 from memgpt.schemas.memgpt_base import MemGPTBase
 from memgpt.schemas.memgpt_message import LegacyMemGPTMessage, MemGPTMessage
-from memgpt.schemas.openai.chat_completions import ToolCall
+from memgpt.schemas.openai.chat_completions import ToolCall, MultiModalMessage
 from memgpt.utils import get_utc_time, is_utc_datetime
 
 
@@ -62,6 +62,7 @@ class Message(BaseMessage):
     id: str = BaseMessage.generate_id_field()
     role: MessageRole = Field(..., description="The role of the participant.")
     text: str = Field(..., description="The text of the message.")
+    content: Union[str, MultiModalMessage] = Field(None, description="Content entered by the user.")
     user_id: str = Field(None, description="The unique identifier of the user.")
     agent_id: str = Field(None, description="The unique identifier of the agent.")
     model: Optional[str] = Field(None, description="The model used to make the function call.")
@@ -223,8 +224,9 @@ def to_openai_dict(
 
         elif self.role == "user":
             assert all([v is not None for v in [self.text, self.role]]), vars(self)
+            content = self.mm_content if self.mm_content is not None else self.text
             openai_message = {
-                "content": self.text,
+                "content": content,
                 "role": self.role,
             }
             # Optional field, do not include if null

memgpt/schemas/openai/chat_completion_request.py

@@ -8,13 +8,15 @@ class SystemMessage(BaseModel):
     role: str = "system"
     name: Optional[str] = None
 
+class MultiModalMessage(BaseModel):
+    type: str
+    image_url: str
 
 class UserMessage(BaseModel):
-    content: Union[str, List[str]]
+    content: Union[str, List[MultiModalMessage]]
     role: str = "user"
     name: Optional[str] = None
 
-
 class ToolCallFunction(BaseModel):
     name: str
     arguments: str

memgpt/schemas/openai/chat_completions.py

@@ -8,13 +8,15 @@ class SystemMessage(BaseModel):
     role: str = "system"
     name: Optional[str] = None
 
+class MultiModalMessage(BaseModel):
+    type: str
+    image_url: str
 
 class UserMessage(BaseModel):
-    content: Union[str, List[str]]
+    content: Union[str, List[MultiModalMessage]]
     role: str = "user"
     name: Optional[str] = None
 
-
 class ToolCallFunction(BaseModel):
     name: str = Field(..., description="The name of the function to call")
     arguments: str = Field(..., description="The arguments to pass to the function (JSON dump)")