Merge pull request #957 from khoj-ai/features/include-full-file-in-co…

…nvo-with-filter

Support including file attachments in the chat message

Now that models have much larger context windows, we can reasonably include full texts of certain files in the messages. Do this when an explicit file filter is set in a conversation. Do so in a separate user message in order to mitigate any confusion in the operation.

Pipe the relevant attached_files context through all methods calling into models.

This breaks certain prior behaviors. We will no longer automatically be processing/generating embeddings on the backend and adding documents to the "brain". You'll have to go to settings and go through the upload documents flow there in order to add docs to the brain (i.e., have search include them during question / response).

src/interface/web/app/chat/page.tsx

@@ -8,7 +8,7 @@ import ChatHistory from "../components/chatHistory/chatHistory";
 import { useSearchParams } from "next/navigation";
 import Loading from "../components/loading/loading";
 
-import { processMessageChunk } from "../common/chatFunctions";
+import { generateNewTitle, processMessageChunk } from "../common/chatFunctions";
 
 import "katex/dist/katex.min.css";
 
@@ -19,7 +19,11 @@ import {
     StreamMessage,
 } from "../components/chatMessage/chatMessage";
 import { useIPLocationData, useIsMobileWidth, welcomeConsole } from "../common/utils";
-import { ChatInputArea, ChatOptions } from "../components/chatInputArea/chatInputArea";
+import {
+    AttachedFileText,
+    ChatInputArea,
+    ChatOptions,
+} from "../components/chatInputArea/chatInputArea";
 import { useAuthenticatedData } from "../common/auth";
 import { AgentData } from "../agents/page";
 
@@ -30,7 +34,7 @@ interface ChatBodyDataProps {
     setQueryToProcess: (query: string) => void;
     streamedMessages: StreamMessage[];
     setStreamedMessages: (messages: StreamMessage[]) => void;
-    setUploadedFiles: (files: string[]) => void;
+    setUploadedFiles: (files: AttachedFileText[] | undefined) => void;
     isMobileWidth?: boolean;
     isLoggedIn: boolean;
     setImages: (images: string[]) => void;
@@ -77,7 +81,24 @@ function ChatBodyData(props: ChatBodyDataProps) {
                 setIsInResearchMode(true);
             }
         }
-    }, [setQueryToProcess, props.setImages]);
+
+        const storedUploadedFiles = localStorage.getItem("uploadedFiles");
+
+        if (storedUploadedFiles) {
+            const parsedFiles = storedUploadedFiles ? JSON.parse(storedUploadedFiles) : [];
+            const uploadedFiles: AttachedFileText[] = [];
+            for (const file of parsedFiles) {
+                uploadedFiles.push({
+                    name: file.name,
+                    file_type: file.file_type,
+                    content: file.content,
+                    size: file.size,
+                });
+            }
+            localStorage.removeItem("uploadedFiles");
+            props.setUploadedFiles(uploadedFiles);
+        }
+    }, [setQueryToProcess, props.setImages, conversationId]);
 
     useEffect(() => {
         if (message) {
@@ -100,6 +121,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
         ) {
             setProcessingMessage(false);
             setImages([]); // Reset images after processing
+            props.setUploadedFiles(undefined); // Reset uploaded files after processing
         } else {
             setMessage("");
         }
@@ -153,7 +175,7 @@ export default function Chat() {
     const [messages, setMessages] = useState<StreamMessage[]>([]);
     const [queryToProcess, setQueryToProcess] = useState<string>("");
     const [processQuerySignal, setProcessQuerySignal] = useState(false);
-    const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
+    const [uploadedFiles, setUploadedFiles] = useState<AttachedFileText[] | undefined>(undefined);
     const [images, setImages] = useState<string[]>([]);
 
     const locationData = useIPLocationData() || {
@@ -192,6 +214,7 @@ export default function Chat() {
                 timestamp: new Date().toISOString(),
                 rawQuery: queryToProcess || "",
                 images: images,
+                queryFiles: uploadedFiles,
             };
             setMessages((prevMessages) => [...prevMessages, newStreamMessage]);
             setProcessQuerySignal(true);
@@ -224,6 +247,9 @@ export default function Chat() {
                 setQueryToProcess("");
                 setProcessQuerySignal(false);
                 setImages([]);
+
+                if (conversationId) generateNewTitle(conversationId, setTitle);
+
                 break;
             }
 
@@ -273,6 +299,7 @@ export default function Chat() {
                 timezone: locationData.timezone,
             }),
             ...(images.length > 0 && { images: images }),
+            ...(uploadedFiles && { files: uploadedFiles }),
         };
 
         const response = await fetch(chatAPI, {
@@ -325,7 +352,7 @@ export default function Chat() {
             <div>
                 <SidePanel
                     conversationId={conversationId}
-                    uploadedFiles={uploadedFiles}
+                    uploadedFiles={[]}
                     isMobileWidth={isMobileWidth}
                 />
             </div>

src/interface/web/app/common/chatFunctions.ts

@@ -267,6 +267,78 @@ export async function createNewConversation(slug: string) {
     }
 }
 
+export async function packageFilesForUpload(files: FileList): Promise<FormData> {
+    const formData = new FormData();
+
+    const fileReadPromises = Array.from(files).map((file) => {
+        return new Promise<void>((resolve, reject) => {
+            let reader = new FileReader();
+            reader.onload = function (event) {
+                if (event.target === null) {
+                    reject();
+                    return;
+                }
+
+                let fileContents = event.target.result;
+                let fileType = file.type;
+                let fileName = file.name;
+                if (fileType === "") {
+                    let fileExtension = fileName.split(".").pop();
+                    if (fileExtension === "org") {
+                        fileType = "text/org";
+                    } else if (fileExtension === "md") {
+                        fileType = "text/markdown";
+                    } else if (fileExtension === "txt") {
+                        fileType = "text/plain";
+                    } else if (fileExtension === "html") {
+                        fileType = "text/html";
+                    } else if (fileExtension === "pdf") {
+                        fileType = "application/pdf";
+                    } else if (fileExtension === "docx") {
+                        fileType =
+                            "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+                    } else {
+                        // Skip this file if its type is not supported
+                        resolve();
+                        return;
+                    }
+                }
+
+                if (fileContents === null) {
+                    reject();
+                    return;
+                }
+
+                let fileObj = new Blob([fileContents], { type: fileType });
+                formData.append("files", fileObj, file.name);
+                resolve();
+            };
+            reader.onerror = reject;
+            reader.readAsArrayBuffer(file);
+        });
+    });
+
+    await Promise.all(fileReadPromises);
+    return formData;
+}
+
+export function generateNewTitle(conversationId: string, setTitle: (title: string) => void) {
+    fetch(`/api/chat/title?conversation_id=${conversationId}`, {
+        method: "POST",
+    })
+        .then((res) => {
+            if (!res.ok) throw new Error(`Failed to call API with error ${res.statusText}`);
+            return res.json();
+        })
+        .then((data) => {
+            setTitle(data.title);
+        })
+        .catch((err) => {
+            console.error(err);
+            return;
+        });
+}
+
 export function uploadDataForIndexing(
     files: FileList,
     setWarning: (warning: string) => void,

src/interface/web/app/common/iconUtils.tsx

@@ -49,8 +49,11 @@ import {
     Gavel,
     Broadcast,
     KeyReturn,
+    FilePdf,
+    FileMd,
+    MicrosoftWordLogo,
 } from "@phosphor-icons/react";
-import { Markdown, OrgMode, Pdf, Word } from "@/app/components/logo/fileLogo";
+import { OrgMode } from "@/app/components/logo/fileLogo";
 
 interface IconMap {
     [key: string]: (color: string, width: string, height: string) => JSX.Element | null;
@@ -238,11 +241,12 @@ function getIconFromFilename(
             return <OrgMode className={className} />;
         case "markdown":
         case "md":
-            return <Markdown className={className} />;
+            return <FileMd className={className} />;
         case "pdf":
-            return <Pdf className={className} />;
+            return <FilePdf className={className} />;
         case "doc":
-            return <Word className={className} />;
+        case "docx":
+            return <MicrosoftWordLogo className={className} />;
         case "jpg":
         case "jpeg":
         case "png":

src/interface/web/app/common/utils.ts

@@ -71,6 +71,16 @@ export function useIsMobileWidth() {
     return isMobileWidth;
 }
 
+export const convertBytesToText = (fileSize: number) => {
+    if (fileSize < 1024) {
+        return `${fileSize} B`;
+    } else if (fileSize < 1024 * 1024) {
+        return `${(fileSize / 1024).toFixed(2)} KB`;
+    } else {
+        return `${(fileSize / (1024 * 1024)).toFixed(2)} MB`;
+    }
+};
+
 export function useDebounce<T>(value: T, delay: number): T {
     const [debouncedValue, setDebouncedValue] = useState<T>(value);
 

src/interface/web/app/components/chatHistory/chatHistory.tsx

@@ -373,6 +373,7 @@ export default function ChatHistory(props: ChatHistoryProps) {
                                             images: message.images,
                                             conversationId: props.conversationId,
                                             turnId: messageTurnId,
+                                            queryFiles: message.queryFiles,
                                         }}
                                         customClassName="fullHistory"
                                         borderLeftColor={`${data?.agent?.color}-500`}