wip(feat/paste): images dropzone (#161)

* WIP(upload files): add a file drag and drop handler for chat. * wip(images): add some ui components for interaction. * wip(images): add attach button. * wip(ui attached images): add a simple ui to see attached files. * refactor(attached images): manage attached images in redux. * wip(images): add images to the user content. * wip(images): send images using kyok and an upstream branch of the lsp `multimodality_plus` * wip(image user messages): render only a string message for now. * wip(images): rendering user images in chat * wip(images): render multiple user images in chat. * wip(images): refactor to reuse the original component for the component is an array. * wip(image): add retry functionality * ui(retry): add buttons for adding and removing images when retrying a question. * ui(images): add a better button for removing the image * feat(images): add error messages for unsupported files. * feat(images): add error and warning handlers for adding files. * refactor(api reset): reset ping first. * typo(dropzone): atttach * ui(image button): change the order for send and attach buttons. * fix(BYOK): title generation response won't have `"metering_balance` when using BYOK. * refactor(images): place images before the user message
smallcloudai · Nov 6, 2024 · 0eb4b8b · 0eb4b8b
1 parent f003f89
commit 0eb4b8b
Show file tree

Hide file tree

Showing 19 changed files with 847 additions and 155 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -100,6 +100,7 @@
     "prettier": "3.1.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
+    "react-dropzone": "^14.2.10",
     "react-markdown": "^9.0.1",
     "react-syntax-highlighter": "^15.5.0",
     "redux-persist": "^6.0.0",

diff --git a/src/app/middleware.ts b/src/app/middleware.ts
@@ -5,8 +5,9 @@ import {
   isRejected,
 } from "@reduxjs/toolkit";
 import {
-  chatAskQuestionThunk,
+  doneStreaming,
   newChatAction,
+  chatAskQuestionThunk,
   restoreChat,
 } from "../features/Chat/Thread";
 import { statisticsApi } from "../services/refact/statistics";
@@ -18,6 +19,7 @@ import { diffApi } from "../services/refact/diffs";
 import { pingApi } from "../services/refact/ping";
 import { clearError, setError } from "../features/Errors/errorsSlice";
 import { updateConfig } from "../features/Config/configSlice";
+import { resetAttachedImagesSlice } from "../features/AttachedImages";
 
 export const listenerMiddleware = createListenerMiddleware();
 const startListening = listenerMiddleware.startListening.withTypes<
@@ -34,13 +36,14 @@ startListening({
   ),
   effect: (_action, listenerApi) => {
     [
+      pingApi.util.resetApiState(),
       statisticsApi.util.resetApiState(),
       capsApi.util.resetApiState(),
       promptsApi.util.resetApiState(),
       toolsApi.util.resetApiState(),
       commandsApi.util.resetApiState(),
       diffApi.util.resetApiState(),
-      pingApi.util.resetApiState(),
+      resetAttachedImagesSlice(),
     ].forEach((api) => listenerApi.dispatch(api));
 
     listenerApi.dispatch(clearError());
@@ -104,3 +107,13 @@ startListening({
     }
   },
 });
+
+startListening({
+  actionCreator: doneStreaming,
+  effect: (action, listenerApi) => {
+    const state = listenerApi.getState();
+    if (action.payload.id === state.chat.thread.id) {
+      listenerApi.dispatch(resetAttachedImagesSlice());
+    }
+  },
+});
diff --git a/src/app/store.ts b/src/app/store.ts
@@ -38,6 +38,7 @@ import { pagesSlice } from "../features/Pages/pagesSlice";
 import mergeInitialState from "redux-persist/lib/stateReconciler/autoMergeLevel2";
 import { listenerMiddleware } from "./middleware";
 import { informationSlice } from "../features/Errors/informationSlice";
+import { attachedImagesSlice } from "../features/AttachedImages";
 
 const tipOfTheDayPersistConfig = {
   key: "totd",
@@ -75,6 +76,7 @@ const rootReducer = combineSlices(
   errorSlice,
   informationSlice,
   pagesSlice,
+  attachedImagesSlice,
 );
 
 const rootPersistConfig = {

diff --git a/src/components/Chat/Chat.tsx b/src/components/Chat/Chat.tsx
@@ -24,6 +24,7 @@ import {
 } from "../../features/Chat/Thread";
 import { ThreadHistoryButton } from "../Buttons";
 import { push } from "../../features/Pages/pagesSlice";
+import { DropzoneProvider } from "../Dropzone";
 import { SystemPrompts } from "../../services/refact";
 
 export type ChatProps = {
@@ -113,72 +114,73 @@ export const Chat: React.FC<ChatProps> = ({
   }, [isWaiting, isStreaming, focusTextarea]);
 
   return (
-    <Flex
-      style={style}
-      direction="column"
-      flexGrow="1"
-      width="100%"
-      overflowY="auto"
-      justify="between"
-      px="1"
-    >
-      <ChatContent
-        key={`chat-content-${chatId}`}
-        ref={chatContentRef}
-        onRetry={retryFromIndex}
-      />
-      {!isStreaming && preventSend && unCalledTools && (
-        <Container py="4" bottom="0" style={{ justifyContent: "flex-end" }}>
-          <Card>
-            <Flex direction="column" align="center" gap="2">
-              Chat was interrupted with uncalled tools calls.
-              <Button onClick={onEnableSend}>Resume</Button>
-            </Flex>
-          </Card>
-        </Container>
-      )}
-
-      <ChatForm
-        key={chatId} // TODO: think of how can we not trigger re-render on chatId change (checkboxes)
-        chatId={chatId}
-        isStreaming={isStreaming}
-        showControls={messages.length === 0 && !isStreaming}
-        onSubmit={handleSummit}
-        model={chatModel}
-        onSetChatModel={onSetChatModel}
-        caps={caps}
-        onStopStreaming={abort}
-        onClose={maybeSendToSidebar}
-        onTextAreaHeightChange={onTextAreaHeightChange}
-        prompts={promptsRequest.data ?? {}}
-        onSetSystemPrompt={onSetSelectedSystemPrompt}
-        selectedSystemPrompt={selectedSystemPrompt}
-      />
-      <Flex justify="between" pl="1" pr="1" pt="1">
-        {/* Two flexboxes are left for the future UI element on the right side */}
-        {messages.length > 0 && (
-          <Flex align="center" justify="between" width="100%">
-            <Flex align="center" gap="1">
-              <Text size="1">model: {chatModel || caps.default_cap} </Text> •{" "}
-              <Text
-                size="1"
-                onClick={() => setIsDebugChatHistoryVisible((prev) => !prev)}
-              >
-                mode: {chatToolUse}{" "}
-              </Text>
-            </Flex>
-            {messages.length !== 0 &&
-              !isStreaming &&
-              isDebugChatHistoryVisible && (
-                <ThreadHistoryButton
-                  title="View history of current thread"
-                  size="1"
-                  onClick={handleThreadHistoryPage}
-                />
-              )}
-          </Flex>
+    <DropzoneProvider asChild>
+      <Flex
+        style={style}
+        direction="column"
+        flexGrow="1"
+        width="100%"
+        overflowY="auto"
+        justify="between"
+        px="1"
+      >
+        <ChatContent
+          key={`chat-content-${chatId}`}
+          ref={chatContentRef}
+          onRetry={retryFromIndex}
+        />
+        {!isStreaming && preventSend && unCalledTools && (
+          <Container py="4" bottom="0" style={{ justifyContent: "flex-end" }}>
+            <Card>
+              <Flex direction="column" align="center" gap="2">
+                Chat was interrupted with uncalled tools calls.
+                <Button onClick={onEnableSend}>Resume</Button>
+              </Flex>
+            </Card>
+          </Container>
         )}
+        <ChatForm
+          key={chatId} // TODO: think of how can we not trigger re-render on chatId change (checkboxes)
+          chatId={chatId}
+          isStreaming={isStreaming}
+          showControls={messages.length === 0 && !isStreaming}
+          onSubmit={handleSummit}
+          model={chatModel}
+          onSetChatModel={onSetChatModel}
+          caps={caps}
+          onStopStreaming={abort}
+          onClose={maybeSendToSidebar}
+          onTextAreaHeightChange={onTextAreaHeightChange}
+          prompts={promptsRequest.data ?? {}}
+          onSetSystemPrompt={onSetSelectedSystemPrompt}
+          selectedSystemPrompt={selectedSystemPrompt}
+        />
+        <Flex justify="between" pl="1" pr="1" pt="1">
+          {/* Two flexboxes are left for the future UI element on the right side */}
+          {messages.length > 0 && (
+            <Flex align="center" justify="between" width="100%">
+              <Flex align="center" gap="1">
+                <Text size="1">model: {chatModel || caps.default_cap} </Text> •{" "}
+                <Text
+                  size="1"
+                  onClick={() => setIsDebugChatHistoryVisible((prev) => !prev)}
+                >
+                  mode: {chatToolUse}{" "}
+                </Text>
+              </Flex>
+              {messages.length !== 0 &&
+                !isStreaming &&
+                isDebugChatHistoryVisible && (
+                  <ThreadHistoryButton
+                    title="View history of current thread"
+                    size="1"
+                    onClick={handleThreadHistoryPage}
+                  />
+                )}
+            </Flex>
+          )}
+        </Flex>
       </Flex>
-    </Flex>
+    </DropzoneProvider>
   );
 };
diff --git a/src/components/ChatContent/ChatContent.tsx b/src/components/ChatContent/ChatContent.tsx
@@ -4,6 +4,7 @@ import {
   isChatContextFileMessage,
   isDiffMessage,
   isToolMessage,
+  UserMessage,
 } from "../../services/refact";
 import { UserInput } from "./UserInput";
 import { ScrollArea } from "../ScrollArea";
@@ -110,7 +111,7 @@ const PlaceHolderText: React.FC = () => {
 };
 
 export type ChatContentProps = {
-  onRetry: (index: number, question: string) => void;
+  onRetry: (index: number, question: UserMessage["content"]) => void;
 };
 
 export const ChatContent = React.forwardRef<HTMLDivElement, ChatContentProps>(
@@ -131,7 +132,10 @@ export const ChatContent = React.forwardRef<HTMLDivElement, ChatContentProps>(
       isStreaming,
     });
 
-    const onRetryWrapper = (index: number, question: string) => {
+    const onRetryWrapper = (
+      index: number,
+      question: UserMessage["content"],
+    ) => {
       props.onRetry(index, question);
       handleScrollButtonClick();
     };
@@ -165,7 +169,7 @@ ChatContent.displayName = "ChatContent";
 
 function renderMessages(
   messages: ChatMessages,
-  onRetry: (index: number, question: string) => void,
+  onRetry: (index: number, question: UserMessage["content"]) => void,
   memo: React.ReactNode[] = [],
   index = 0,
 ) {
@@ -197,6 +201,7 @@ function renderMessages(
 
   if (head.role === "user") {
     const key = "user-input-" + index;
+
     const nextMemo = [
       ...memo,
       <UserInput onRetry={onRetry} key={key} messageIndex={index}>