feat: select the corresponding parsing method according to the file t…

…ype and after the document is successfully uploaded, use the ChunkMethodModal to select the parsing method. and remove ChunkMethodModal from knowledge-file (infiniflow#158) * feat: select the corresponding parsing method according to the file type * feat: after the document is successfully uploaded, use the ChunkMethodModal to select the parsing method. * feat: add pdf types to ParserListMap * feat: remove ChunkMethodModal from knowledge-file
Halfknow · Mar 27, 2024 · c72d31b · c72d31b
1 parent a8bc4c1
commit c72d31b
Show file tree

Hide file tree

Showing 14 changed files with 340 additions and 219 deletions.
diff --git a/web/src/components/chunk-method-modal/hooks.ts b/web/src/components/chunk-method-modal/hooks.ts
@@ -0,0 +1,82 @@
+import {
+  useFetchTenantInfo,
+  useSelectParserList,
+} from '@/hooks/userSettingHook';
+import { useEffect, useMemo, useState } from 'react';
+
+const ParserListMap = new Map([
+  [
+    ['pdf'],
+    [
+      'naive',
+      'resume',
+      'manual',
+      'paper',
+      'book',
+      'laws',
+      'presentation',
+      'one',
+    ],
+  ],
+  [
+    ['doc', 'docx'],
+    ['naive', 'resume', 'book', 'laws', 'one'],
+  ],
+  [
+    ['xlsx', 'xls'],
+    ['naive', 'qa', 'table', 'one'],
+  ],
+  [['ppt', 'pptx'], ['presentation']],
+  [
+    ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tif', 'tiff', 'webp', 'svg', 'ico'],
+    ['picture'],
+  ],
+  [['txt'], ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table']],
+  [['csv'], ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table']],
+]);
+
+const getParserList = (
+  values: string[],
+  parserList: Array<{
+    value: string;
+    label: string;
+  }>,
+) => {
+  return parserList.filter((x) => values?.some((y) => y === x.value));
+};
+
+export const useFetchParserListOnMount = (
+  parserId: string,
+  documentExtension: string,
+) => {
+  const [selectedTag, setSelectedTag] = useState('');
+  const parserList = useSelectParserList();
+
+  const nextParserList = useMemo(() => {
+    const key = [...ParserListMap.keys()].find((x) =>
+      x.some((y) => y === documentExtension),
+    );
+    if (key) {
+      const values = ParserListMap.get(key);
+      return getParserList(values ?? [], parserList);
+    }
+
+    return getParserList(
+      ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table'],
+      parserList,
+    );
+  }, [parserList, documentExtension]);
+
+  useFetchTenantInfo();
+
+  useEffect(() => {
+    setSelectedTag(parserId);
+  }, [parserId]);
+
+  const handleChange = (tag: string, checked: boolean) => {
+    const nextSelectedTag = checked ? tag : selectedTag;
+    setSelectedTag(nextSelectedTag);
+  };
+
+  return { parserList: nextParserList, handleChange, selectedTag };
+};
diff --git a/web/src/components/chunk-method-modal/index.less b/web/src/components/chunk-method-modal/index.less
@@ -0,0 +1,10 @@
+.pageInputNumber {
+  width: 220px;
+}
+
+.questionIcon {
+  margin-inline-start: 4px;
+  color: rgba(0, 0, 0, 0.45);
+  cursor: help;
+  writing-mode: horizontal-tb;
+}
diff --git a/...nts/knowledge-file/chunk-method-modal.tsx → ...c/components/chunk-method-modal/index.tsx b/...nts/knowledge-file/chunk-method-modal.tsx → ...c/components/chunk-method-modal/index.tsx
@@ -35,7 +35,7 @@ interface IProps extends Omit<IModalManagerChildrenProps, 'showModal'> {
   showModal?(): void;
   parserId: string;
   parserConfig: IKnowledgeFileParserConfig;
-  documentType: string;
+  documentExtension: string;
 }
 
 const hidePagesChunkMethods = ['qa', 'table', 'picture', 'resume', 'one'];
@@ -45,11 +45,13 @@ const ChunkMethodModal: React.FC<IProps> = ({
   onOk,
   hideModal,
   visible,
-  documentType,
+  documentExtension,
   parserConfig,
 }) => {
-  const { parserList, handleChange, selectedTag } =
-    useFetchParserListOnMount(parserId);
+  const { parserList, handleChange, selectedTag } = useFetchParserListOnMount(
+    parserId,
+    documentExtension,
+  );
   const [form] = Form.useForm();
 
   const handleOk = async () => {
@@ -62,11 +64,8 @@ const ChunkMethodModal: React.FC<IProps> = ({
   };
 
   const showPages = useMemo(() => {
-    return (
-      documentType === 'pdf' &&
-      hidePagesChunkMethods.every((x) => x !== selectedTag)
-    );
-  }, [documentType, selectedTag]);
+    return hidePagesChunkMethods.every((x) => x !== selectedTag);
+  }, [selectedTag]);
 
   const showOne = useMemo(() => {
     return showPages || selectedTag === 'one';
@@ -114,7 +113,7 @@ const ChunkMethodModal: React.FC<IProps> = ({
       </Space>
       <Divider></Divider>
 
-      {
+      {documentExtension === 'pdf' && (
         <Form name="dynamic_form_nest_item" autoComplete="off" form={form}>
           {showPages && (
             <>
@@ -271,7 +270,7 @@ const ChunkMethodModal: React.FC<IProps> = ({
 
           {selectedTag === 'naive' && <MaxTokenNumber></MaxTokenNumber>}
         </Form>
-      }
+      )}
     </Modal>
   );
 };

diff --git a/web/src/hooks/documentHooks.ts b/web/src/hooks/documentHooks.ts
@@ -2,6 +2,7 @@ import { IChunk, IKnowledgeFile } from '@/interfaces/database/knowledge';
 import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
 import { api_host } from '@/utils/api';
 import { buildChunkHighlights } from '@/utils/documentUtils';
+import { UploadFile } from 'antd';
 import { useCallback, useMemo, useState } from 'react';
 import { IHighlight } from 'react-pdf-highlighter';
 import { useDispatch, useSelector } from 'umi';
@@ -174,3 +175,47 @@ export const useRemoveDocument = (documentId: string) => {
 
   return removeDocument;
 };
+
+export const useUploadDocument = () => {
+  const dispatch = useDispatch();
+  const { knowledgeId } = useGetKnowledgeSearchParams();
+
+  const uploadDocument = useCallback(
+    (file: UploadFile) => {
+      try {
+        return dispatch<any>({
+          type: 'kFModel/upload_document',
+          payload: {
+            file,
+            kb_id: knowledgeId,
+          },
+        });
+      } catch (errorInfo) {
+        console.log('Failed:', errorInfo);
+      }
+    },
+    [dispatch, knowledgeId],
+  );
+
+  return uploadDocument;
+};
+
+export const useRunDocument = () => {
+  const dispatch = useDispatch();
+
+  const runDocumentByIds = useCallback(
+    (ids: string[]) => {
+      try {
+        return dispatch<any>({
+          type: 'kFModel/document_run',
+          payload: { doc_ids: ids, run: 1 },
+        });
+      } catch (errorInfo) {
+        console.log('Failed:', errorInfo);
+      }
+    },
+    [dispatch],
+  );
+
+  return runDocumentByIds;
+};
diff --git a/web/src/hooks/logicHooks.ts b/web/src/hooks/logicHooks.ts
@@ -0,0 +1,47 @@
+import { IKnowledgeFile } from '@/interfaces/database/knowledge';
+import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
+import { useCallback, useState } from 'react';
+import { useSetModalState } from './commonHooks';
+import { useSetDocumentParser } from './documentHooks';
+import { useOneNamespaceEffectsLoading } from './storeHooks';
+
+export const useChangeDocumentParser = (documentId: string) => {
+  const setDocumentParser = useSetDocumentParser();
+
+  const {
+    visible: changeParserVisible,
+    hideModal: hideChangeParserModal,
+    showModal: showChangeParserModal,
+  } = useSetModalState();
+  const loading = useOneNamespaceEffectsLoading('kFModel', [
+    'document_change_parser',
+  ]);
+
+  const onChangeParserOk = useCallback(
+    async (parserId: string, parserConfig: IChangeParserConfigRequestBody) => {
+      const ret = await setDocumentParser(parserId, documentId, parserConfig);
+      if (ret === 0) {
+        hideChangeParserModal();
+      }
+    },
+    [hideChangeParserModal, setDocumentParser, documentId],
+  );
+
+  return {
+    changeParserLoading: loading,
+    onChangeParserOk,
+    changeParserVisible,
+    hideChangeParserModal,
+    showChangeParserModal,
+  };
+};
+
+export const useSetSelectedRecord = <T = IKnowledgeFile>() => {
+  const [currentRecord, setCurrentRecord] = useState<T>({} as T);
+
+  const setRecord = (record: T) => {
+    setCurrentRecord(record);
+  };
+
+  return { currentRecord, setRecord };
+};