Hetao datasource (#511)

* feat: add datasource in import * feat: add multiple datasources * mod: update page * feat: add importer config * feat: add load directory of datasource * mod: update file select logic before import * feat: support concat id & edit config * feat: get config from db before import & fix some issues * mod: add platform in config && modify datasource model * mod: adjust the region rules in the config rules * feat: support cos platform
vesoft-inc · Mar 24, 2023 · a20ba63 · a20ba63
1 parent e824a60
commit a20ba63
Show file tree

Hide file tree

Showing 64 changed files with 2,504 additions and 515 deletions.
diff --git a/app/app.less b/app/app.less
@@ -43,7 +43,6 @@
 	display: flex;
 	justify-content: center;
 	padding-bottom: 16px;
-	border-bottom: 1px solid @gray;
 }
 
 .ant-radio-group.studioTabGroup {

diff --git a/app/components/CSVPreviewLink/index.module.less b/app/components/CSVPreviewLink/index.module.less
@@ -44,11 +44,15 @@
 	}
 
 	> .operation {
+		display: flex;
 		text-align: center;
 		position: absolute;
     bottom: 15px;
     left: 50%;
     transform: translateX(-50%);
+		:global(.ant-btn:not(:last-child)) {
+			margin-right: 23px;
+		}
 	}
 
 	.anticon {

diff --git a/app/components/CSVPreviewLink/index.tsx b/app/components/CSVPreviewLink/index.tsx
@@ -1,26 +1,30 @@
 import { Button, Popover, Table } from 'antd';
-import React, { useEffect, useState } from 'react';
+import React, { useCallback, useEffect, useState } from 'react';
 import { useI18n } from '@vesoft-inc/i18n';
 import { v4 as uuidv4 } from 'uuid';
 import cls from 'classnames';
 import { usePapaParse } from 'react-papaparse';
 import { StudioFile } from '@app/interfaces/import';
+import { CheckOutlined } from '@ant-design/icons';
 import styles from './index.module.less';
 
 interface IProps {
   file: StudioFile;
   children: any;
-  onMapping?: (index: number) => void;
+  onMapping?: (index: number[] | number) => void;
   btnType?: string
   selected?: boolean
+  multipleMode?: boolean,
+  data: number[] | number
 }
 
 const CSVPreviewLink = (props: IProps) => {
-  const { onMapping, file, children, btnType, selected } = props;
+  const { onMapping, file, children, btnType, selected, multipleMode, data } = props;
   const [visible, setVisible] = useState(false);
-  const [data, setData] = useState<any[]>([]);
+  const [datasource, setDatasource] = useState<any[]>([]);
   const { intl } = useI18n();
   const { readString } = usePapaParse();
+  const [indexes, setIndexes] = useState<number[]>([]);
   useEffect(() => {
     if(!file) return;
     const { delimiter, sample } = file;
@@ -33,40 +37,67 @@ const CSVPreviewLink = (props: IProps) => {
         data = [...data, row.data];
       },
       complete: () => {
-        setData(data);
+        setDatasource(data);
       } 
     });
   }, [file]);
-  const handleLinkClick = e => {
+
+  useEffect(() => {
+    setIndexes(data ? (Array.isArray(data) ? data : [data]) : []);
+  }, [data]);
+  const handleLinkClick = useCallback(e => {
     e.stopPropagation();
     setVisible(true);
-  };
-  const handleMapping = (index: number, e: React.MouseEvent) => {
+  }, []);
+  const handleMapping = useCallback((e) => {
+    e.stopPropagation();
+    onMapping?.(multipleMode ? indexes : indexes[0]);
+    setVisible(false);
+  }, [indexes, onMapping]);
+  const handleClear = useCallback((e) => {
     e.stopPropagation();
-    onMapping?.(index);
+    onMapping?.(null);
     setVisible(false);
-  };
-  const columns = data[0]?.map((header, index) => {
+    setIndexes([]);
+  }, [onMapping]);
+
+  const toggleMapping = useCallback((index: number, e: React.MouseEvent) => {
+    e.stopPropagation();
+    if(!multipleMode) {
+      setIndexes([index]);
+      return;
+    }
+    const _indexes = [...indexes];
+    if(indexes.indexOf(index) > -1) {
+      _indexes.splice(indexes.indexOf(index), 1);
+    } else {
+      _indexes.push(index);
+    }
+    setIndexes(_indexes);
+  }, [multipleMode, indexes]);
+
+  const columns = datasource[0]?.map((header, index) => {
     const textIndex = index;
     const _header = file?.withHeader ? header : `Column ${textIndex}`;
+    const isSelected = indexes.indexOf(textIndex) > -1;
     return {
       title: onMapping ? (
         <Button
-          type="primary"
+          type={isSelected ? 'primary' : 'default'}
           className={styles.csvSelectIndex}
-          onClick={(e) => handleMapping(textIndex, e)}
-        >{_header}</Button>
+          onClick={(e) => toggleMapping(textIndex, e)}
+        >{isSelected && <CheckOutlined />}{_header}</Button>
       ) : (
         _header
       ),
       dataIndex: index,
       render: value => <span className={styles.limitWidth}>{value}</span>,
     };
   }) || [];
-  const handleOpen = (visible) => {
+  const handleOpen = useCallback((visible) => {
     if(!file) return;
     setVisible(visible);
-  };
+  }, [file]);
   return (
     <Popover
       destroyTooltipOnHide={true}
@@ -80,16 +111,21 @@ const CSVPreviewLink = (props: IProps) => {
         <Table
           bordered={false}
           className={cls({ [styles.noBackground]: !!onMapping })}
-          dataSource={file?.withHeader ? data.slice(1) : data}
+          dataSource={file?.withHeader ? datasource.slice(1) : datasource}
           columns={columns}
           pagination={false}
           rowKey={() => uuidv4()}
         />
         <div className={styles.operation}>
           {onMapping && (
-            <Button onClick={(e) => handleMapping(null, e)} className="primaryBtn studioAddBtn">
-              {intl.get('import.ignore')}
-            </Button>
+            <>
+              <Button onClick={handleClear} className="primaryBtn studioAddBtn">
+                {intl.get('import.ignore')}
+              </Button>
+              <Button type="primary" onClick={handleMapping}>
+                {intl.get('common.confirm')}
+              </Button>
+            </>
           )}
         </div>
       </div>}

diff --git a/...eList/UploadConfigModal/index.module.less → ...nents/FileConfigSetting/index.module.less b/...eList/UploadConfigModal/index.module.less → ...nents/FileConfigSetting/index.module.less
@@ -1,10 +1,5 @@
 @import '~@app/common.less';
 
-.uploadModal {
-  :global(.ant-modal-body) {
-		padding: 0 0 20px;
-	}
-}
 .container {
   display: flex;
   border-bottom: 1px solid #D5DDEB;

diff --git a/...port/FileList/UploadConfigModal/index.tsx → app/components/FileConfigSetting/index.tsx b/...port/FileList/UploadConfigModal/index.tsx → app/components/FileConfigSetting/index.tsx
@@ -1,22 +1,21 @@
 import Icon from '@app/components/Icon';
 import { useI18n } from '@vesoft-inc/i18n';
-import { Button, Input, Modal, Table, Popconfirm, Dropdown, message } from 'antd';
+import { Button, Input, Modal, Table, Popconfirm, Dropdown } from 'antd';
 import { v4 as uuidv4 } from 'uuid';
 import React, { useCallback, useEffect, useState } from 'react';
 import { usePapaParse } from 'react-papaparse';
 import cls from 'classnames';
 import { StudioFile } from '@app/interfaces/import';
-import { useStore } from '@app/stores';
 import { observer, useLocalObservable } from 'mobx-react-lite';
 import { ExclamationCircleFilled } from '@ant-design/icons';
 import { observable } from 'mobx';
 import Checkbox, { CheckboxChangeEvent } from 'antd/lib/checkbox';
 import styles from './index.module.less';
 interface IProps {
-  visible: boolean;
-  onConfirm: () => void;
+  onConfirm: (data) => void;
   onCancel: () => void;
-  uploadList: StudioFile[];
+  preUploadList: StudioFile[];
+  duplicateCheckList?: StudioFile[];
 }
 
 const DelimiterConfigModal = (props: { onConfirm: (string) => void }) => {
@@ -30,10 +29,8 @@ const DelimiterConfigModal = (props: { onConfirm: (string) => void }) => {
     </div>
   );
 };
-const UploadConfigModal = (props: IProps) => {
-  const { visible, onConfirm, onCancel, uploadList } = props;
-  const { files } = useStore();
-  const { fileList, uploadFile } = files;
+const FileConfigSetting = (props: IProps) => {
+  const { onConfirm, onCancel, preUploadList, duplicateCheckList } = props;
   const { intl } = useI18n();
   const state = useLocalObservable(() => ({
     data: [],
@@ -44,34 +41,54 @@ const UploadConfigModal = (props: IProps) => {
     loading: false,
     uploading: false,
     setState: (obj) => Object.assign(state, obj),
-  }), { data: observable.ref });
-  const { readRemoteFile } = usePapaParse();
+  }), { data: observable.ref, activeItem: observable.ref });
+  const { readRemoteFile, readString } = usePapaParse();
   useEffect(() => {
     const { setState } = state;
-    visible && setState({ data: uploadList, activeItem: uploadList[0] });
-  }, [visible]);
+    setState({
+      data: preUploadList, 
+      activeItem: preUploadList[0], 
+      checkAll: preUploadList.every((item) => item.withHeader),
+      indeterminate: preUploadList.some((item) => item.withHeader) && !preUploadList.every((item) => item.withHeader),
+    });
+  }, []);
   useEffect(() => {
     state.activeItem && readFile();
   }, [state.activeItem]);
   const readFile = useCallback(() => {
     const { activeItem, setState } = state;
     if(!activeItem) return;
     setState({ loading: true });
-    const url = URL.createObjectURL(activeItem);
     let content = [];
-    readRemoteFile(url, { 
-      delimiter: activeItem.delimiter, 
-      download: true, 
-      preview: 5,
-      worker: true, 
-      skipEmptyLines: true,
-      step: (row) => {
-        content = [...content, row.data];
-      },
-      complete: () => {
-        setState({ loading: false, previewContent: content });
-      } 
-    });
+    if(activeItem.sample) {
+      readString(activeItem.sample, { 
+        delimiter: activeItem.delimiter || ',',
+        worker: true, 
+        skipEmptyLines: true,
+        step: (row) => {
+          content = [...content, row.data];
+        },
+        complete: () => {
+          setState({ loading: false, previewContent: content });
+        } 
+      });
+    } else {
+      const url = URL.createObjectURL(activeItem);
+      readRemoteFile(url, { 
+        delimiter: activeItem.delimiter, 
+        download: true, 
+        preview: 5,
+        worker: true, 
+        skipEmptyLines: true,
+        step: (row) => {
+          content = [...content, row.data];
+        },
+        complete: () => {
+          setState({ loading: false, previewContent: content });
+        } 
+      });
+    }
+
   }, []);
 
   const onCheckAllChange = useCallback((e: CheckboxChangeEvent) => {
@@ -80,7 +97,7 @@ const UploadConfigModal = (props: IProps) => {
     setState({
       checkAll: checked,
       indeterminate: false,
-      data: data.map(i => (i.withHeader = checked, i))
+      data: data.map(i => (i.withHeader = checked, i)),
     });
   }, []);
 
@@ -127,7 +144,7 @@ const UploadConfigModal = (props: IProps) => {
 
   const handleConfirm = useCallback(() => {
     const { data } = state;
-    const existFileName = fileList.map((file) => file.name);
+    const existFileName = duplicateCheckList?.map((file) => file.name) || [];
     const repeatFiles = data.filter((file) => existFileName.includes(file.name));
     if(!repeatFiles.length) {
       startImport();
@@ -152,25 +169,18 @@ const UploadConfigModal = (props: IProps) => {
         startImport();
       },
     });
-  }, [fileList]);
+  }, [duplicateCheckList]);
   const startImport = useCallback(async () => {
     const { data, setState } = state;
     setState({ uploading: true });
-    const res = await uploadFile(data);
-    if(res.code === 0) {
-      onConfirm();
-      message.success(intl.get('import.uploadSuccessfully'));
-    }
+    await onConfirm(data);
     setState({ uploading: false });
   }, []);
   const handleCancel = useCallback(() => {
     const { uploading } = state;
     !uploading && onCancel();
   }, []);
 
-  if(!visible) {
-    return null;
-  }
   const { uploading, data, activeItem, previewContent, loading, setState, checkAll, indeterminate } = state;
   const parseColumns = previewContent.length
     ? previewContent[0].map((header, index) => {
@@ -229,16 +239,8 @@ const UploadConfigModal = (props: IProps) => {
       ),
     },
   ];
-
   return (
-    <Modal
-      title={intl.get('import.previewFiles')}
-      open={visible}
-      width={920}
-      onCancel={() => handleCancel()}
-      className={styles.uploadModal}
-      footer={false}
-    >
+    <div>
       <div className={styles.container}>
         <div className={styles.left}>
           <Table
@@ -254,7 +256,7 @@ const UploadConfigModal = (props: IProps) => {
             className={styles.previewTable}
             dataSource={data}
             columns={columns}
-            rowKey="uid"
+            rowKey={() => uuidv4()}
             pagination={false}
           />
         </div>
@@ -286,8 +288,9 @@ const UploadConfigModal = (props: IProps) => {
           {intl.get('common.confirm')}
         </Button>
       </div>
-    </Modal>
+    </div>
   );
 };
 
-export default observer(UploadConfigModal);
+
+export default observer(FileConfigSetting);