From 42865868b9c29afb154976a5c6d65684570681b7 Mon Sep 17 00:00:00 2001 From: hetao92 <18328704+hetao92@users.noreply.github.com> Date: Thu, 9 Mar 2023 08:57:24 +0000 Subject: [PATCH 01/11] feat: add datasource in import --- app/app.less | 1 - app/config/locale/en-US.json | 15 ++- app/config/locale/zh-CN.json | 15 ++- app/interfaces/import.ts | 13 +++ .../PreviewFileModal/index.module.less | 0 .../LocalFileList}/PreviewFileModal/index.tsx | 0 .../UploadConfigModal/index.module.less | 0 .../UploadConfigModal/index.tsx | 0 .../LocalFileList}/index.module.less | 0 .../LocalFileList}/index.tsx | 2 +- .../RemoteList/index.module.less | 35 ++++++ .../DataSourceList/RemoteList/index.tsx | 102 ++++++++++++++++++ .../Import/DataSourceList/index.module.less | 9 ++ app/pages/Import/DataSourceList/index.tsx | 36 +++++++ .../Import/TaskList/TemplateModal/index.tsx | 2 +- app/pages/Import/TaskList/index.module.less | 3 +- app/pages/Import/index.tsx | 12 +-- app/pages/MainPage/routes.tsx | 2 +- .../Schema/SchemaConfig/index.module.less | 3 + app/pages/Welcome/index.tsx | 2 +- 20 files changed, 238 insertions(+), 14 deletions(-) rename app/pages/Import/{FileList => DataSourceList/LocalFileList}/PreviewFileModal/index.module.less (100%) rename app/pages/Import/{FileList => DataSourceList/LocalFileList}/PreviewFileModal/index.tsx (100%) rename app/pages/Import/{FileList => DataSourceList/LocalFileList}/UploadConfigModal/index.module.less (100%) rename app/pages/Import/{FileList => DataSourceList/LocalFileList}/UploadConfigModal/index.tsx (100%) rename app/pages/Import/{FileList => DataSourceList/LocalFileList}/index.module.less (100%) rename app/pages/Import/{FileList => DataSourceList/LocalFileList}/index.tsx (99%) create mode 100644 app/pages/Import/DataSourceList/RemoteList/index.module.less create mode 100644 app/pages/Import/DataSourceList/RemoteList/index.tsx create mode 100644 app/pages/Import/DataSourceList/index.module.less create mode 100644 app/pages/Import/DataSourceList/index.tsx diff --git a/app/app.less b/app/app.less index c463b20b..b1e9e025 100644 --- a/app/app.less +++ b/app/app.less @@ -43,7 +43,6 @@ display: flex; justify-content: center; padding-bottom: 16px; - border-bottom: 1px solid @gray; } .ant-radio-group.studioTabGroup { diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json index a8fb7149..68c1effc 100644 --- a/app/config/locale/en-US.json +++ b/app/config/locale/en-US.json @@ -153,6 +153,7 @@ }, "import": { "uploadFile": "Upload Files", + "dataSourceManagement": "Data Source Management", "importData": "Import Data", "createTask": "New Import", "uploadTemp": "Import Template", @@ -250,7 +251,19 @@ "deleteFiles": "Delete select files", "fileRepeatTip": "These files are already exists, continuing to upload will overwrite the original file", "filePreview": "Preview file {name}", - "uploadConfirm": "Upload Confirm" + "uploadConfirm": "Upload Confirm", + "localFiles": "Local files", + "cloudStorage": "Cloud storage", + "sftp": "SFTP", + "newDataSource": "New Data Source", + "deleteDataSource": "Delete Data Source", + "dataSourceList": "{type} list", + "ipAddress": "IP Address:Port", + "bucketName": "Bucket Name", + "accessKeyId": "AccessKeyId", + "region": "Region", + "addDate": "Add Date", + "account": "Account" }, "schema": { "spaceList": "Graph Space List", diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json index 21991f22..8d5b22c1 100644 --- a/app/config/locale/zh-CN.json +++ b/app/config/locale/zh-CN.json @@ -153,6 +153,7 @@ }, "import": { "uploadFile": "上传文件", + "dataSourceManagement": "数据源管理", "importData": "导入数据", "createTask": "创建导入任务", "uploadTemp": "导入模板", @@ -250,7 +251,19 @@ "deleteFiles": "删除选中文件", "fileRepeatTip": "上述文件已存在，继续上传将覆盖原文件", "filePreview": "预览文件 {name}", - "uploadConfirm": "上传文件确认" + "uploadConfirm": "上传文件确认", + "localFiles": "本地文件", + "cloudStorage": "云存储", + "sftp": "SFTP", + "newDataSource": "新建数据源", + "deleteDataSource": "删除数据源", + "dataSourceList": "{type}列表", + "ipAddress": "IP 地址：端口", + "bucketName": "Bucket 名称", + "accessKeyId": "AccessKeyId", + "region": "区域", + "addDate": "添加日期", + "account": "账号" }, "schema": { "spaceList": "图空间列表", diff --git a/app/interfaces/import.ts b/app/interfaces/import.ts index 7201e3a7..b17114f5 100644 --- a/app/interfaces/import.ts +++ b/app/interfaces/import.ts @@ -69,4 +69,17 @@ export interface StudioFile extends RcFile { delimiter?: string; sample?: string; content?: any[]; +} + +export enum IRemoteType { + 'Cloud' = 'cloudStorage', + 'Sftp' = 'sftp', +} + +export interface ICloudStorage { + ipAddress: string; + bucketName: string; + accessKeyId: string; + region: string; + addDate: number; } \ No newline at end of file diff --git a/app/pages/Import/FileList/PreviewFileModal/index.module.less b/app/pages/Import/DataSourceList/LocalFileList/PreviewFileModal/index.module.less similarity index 100% rename from app/pages/Import/FileList/PreviewFileModal/index.module.less rename to app/pages/Import/DataSourceList/LocalFileList/PreviewFileModal/index.module.less diff --git a/app/pages/Import/FileList/PreviewFileModal/index.tsx b/app/pages/Import/DataSourceList/LocalFileList/PreviewFileModal/index.tsx similarity index 100% rename from app/pages/Import/FileList/PreviewFileModal/index.tsx rename to app/pages/Import/DataSourceList/LocalFileList/PreviewFileModal/index.tsx diff --git a/app/pages/Import/FileList/UploadConfigModal/index.module.less b/app/pages/Import/DataSourceList/LocalFileList/UploadConfigModal/index.module.less similarity index 100% rename from app/pages/Import/FileList/UploadConfigModal/index.module.less rename to app/pages/Import/DataSourceList/LocalFileList/UploadConfigModal/index.module.less diff --git a/app/pages/Import/FileList/UploadConfigModal/index.tsx b/app/pages/Import/DataSourceList/LocalFileList/UploadConfigModal/index.tsx similarity index 100% rename from app/pages/Import/FileList/UploadConfigModal/index.tsx rename to app/pages/Import/DataSourceList/LocalFileList/UploadConfigModal/index.tsx diff --git a/app/pages/Import/FileList/index.module.less b/app/pages/Import/DataSourceList/LocalFileList/index.module.less similarity index 100% rename from app/pages/Import/FileList/index.module.less rename to app/pages/Import/DataSourceList/LocalFileList/index.module.less diff --git a/app/pages/Import/FileList/index.tsx b/app/pages/Import/DataSourceList/LocalFileList/index.tsx similarity index 99% rename from app/pages/Import/FileList/index.tsx rename to app/pages/Import/DataSourceList/LocalFileList/index.tsx index 12374e4d..62ed2819 100644 --- a/app/pages/Import/FileList/index.tsx +++ b/app/pages/Import/DataSourceList/LocalFileList/index.tsx @@ -102,7 +102,7 @@ const FileList = () => { }, [selectFiles]); useEffect(() => { getFileList(); - trackPageView('/import/files'); + trackPageView('/import/dataSources'); }, []); return (
diff --git a/app/pages/Import/DataSourceList/RemoteList/index.module.less b/app/pages/Import/DataSourceList/RemoteList/index.module.less new file mode 100644 index 00000000..0541aded --- /dev/null +++ b/app/pages/Import/DataSourceList/RemoteList/index.module.less @@ -0,0 +1,35 @@ +@import '~@app/common.less'; +.fileUpload { + .uploadBtn { + margin: 15px 0; + } + .fileOperations { + display: flex; + align-items: center; + .deleteBtn { + margin-left: 15px; + } + } + .fileList { + margin-top: 10px; + h3 { + color: @lightBlack; + font-weight: bold; + font-size: 18px; + padding-bottom: 12px; + border-bottom: 1px solid @gray; + margin-bottom: 20px; + } + } + .operation { + display: flex; + align-items: center; + button { + padding: 8px 24px; + justify-content: flex-start; + } + button:not(:last-child) { + margin-right: 30px; + } + } +} \ No newline at end of file diff --git a/app/pages/Import/DataSourceList/RemoteList/index.tsx b/app/pages/Import/DataSourceList/RemoteList/index.tsx new file mode 100644 index 00000000..447975ff --- /dev/null +++ b/app/pages/Import/DataSourceList/RemoteList/index.tsx @@ -0,0 +1,102 @@ +import React, { useCallback, useEffect, useState } from 'react'; +import { observer } from 'mobx-react-lite'; +import { useStore } from '@app/stores'; +import { trackPageView } from '@app/utils/stat'; +import { Button, Popconfirm, Table } from 'antd'; +import Icon from '@app/components/Icon'; +import cls from 'classnames'; +import { IRemoteType } from '@app/interfaces/import'; +import { useI18n } from '@vesoft-inc/i18n'; + +import styles from './index.module.less'; + +interface IProps { + type: IRemoteType; +} +const cloudKeys = ['ipAddress', 'bucketName', 'accessKeyId', 'region', 'addDate']; +const sftpKeys = ['ipAddress', 'account', 'addDate']; +const columnKeys = { + [IRemoteType.Cloud]: cloudKeys, + [IRemoteType.Sftp]: sftpKeys, +}; +const FileList = (props: IProps) => { + const { type } = props; + const { files, global } = useStore(); + const { intl } = useI18n(); + const { fileList, deleteFile, getFiles } = files; + const [loading, setLoading] = useState(false); + const [selectItems, setSelectItems] = useState([]); + const columns = columnKeys[type].map(key => ({ + title: intl.get(`import.${key}`), + dataIndex: key, + })).concat(({ + title: intl.get('common.operation'), + key: 'operation', + render: (_, file) => (
+ + deleteFile([file.name])} + title={intl.get('common.ask')} + okText={intl.get('common.confirm')} + cancelText={intl.get('common.cancel')} + > + + +
) + } as any)); + + + const getFileList = async () => { + !loading && setLoading(true); + await getFiles(); + setLoading(false); + }; + const handleDeleteFiles = useCallback(async () => { + const flag = await deleteFile(selectItems); + flag && setSelectItems([]); + }, [selectItems]); + useEffect(() => { + getFileList(); + trackPageView('/import/dataSources'); + }, []); + return ( +
+
+ + + + +
+
+

{intl.get('import.dataSourceList', { type: intl.get(`import.${type}`) })} ({fileList.length})

+ setSelectItems(selectedRowKeys as string[]), + }} + columns={columns} + rowKey="name" + pagination={false} + /> + + + ); +}; + +export default observer(FileList); diff --git a/app/pages/Import/DataSourceList/index.module.less b/app/pages/Import/DataSourceList/index.module.less new file mode 100644 index 00000000..c8fc398e --- /dev/null +++ b/app/pages/Import/DataSourceList/index.module.less @@ -0,0 +1,9 @@ +@import '~@app/common.less'; +.dataSourceContainer { + .sourceTabs { + :global(.ant-tabs-tab-btn) { + font-size: 18px; + padding: 0 20px; + } + } +} \ No newline at end of file diff --git a/app/pages/Import/DataSourceList/index.tsx b/app/pages/Import/DataSourceList/index.tsx new file mode 100644 index 00000000..763f8915 --- /dev/null +++ b/app/pages/Import/DataSourceList/index.tsx @@ -0,0 +1,36 @@ +import React from 'react'; +import { observer } from 'mobx-react-lite'; +import { Tabs, TabsProps } from 'antd'; +import { useI18n } from '@vesoft-inc/i18n'; +import { IRemoteType } from '@app/interfaces/import'; +import LocalFileList from './LocalFileList'; +import RemoteList from './RemoteList'; +import styles from './index.module.less'; + +const DataSourceList = () => { + const { intl } = useI18n(); + const items: TabsProps['items'] = [ + { + key: 'local', + label: intl.get('import.localFiles'), + children: + }, + { + key: 'cloud', + label: intl.get('import.cloudStorage'), + children: + }, + { + key: 'sftp', + label: intl.get('import.sftp'), + children: + }, + ]; + return ( +
+ +
+ ); +}; + +export default observer(DataSourceList); diff --git a/app/pages/Import/TaskList/TemplateModal/index.tsx b/app/pages/Import/TaskList/TemplateModal/index.tsx index 5de9538d..68f3627c 100644 --- a/app/pages/Import/TaskList/TemplateModal/index.tsx +++ b/app/pages/Import/TaskList/TemplateModal/index.tsx @@ -118,7 +118,7 @@ const TemplateModal = (props: IProps) => { {!config ?

{intl.get('import.fileUploadRequired')} - {intl.get('import.uploadFile')} + {intl.get('import.uploadFile')} {intl.get('import.fileUploadRequired2')}

diff --git a/app/pages/Import/TaskList/index.module.less b/app/pages/Import/TaskList/index.module.less index bd8061f1..489a7d9d 100644 --- a/app/pages/Import/TaskList/index.module.less +++ b/app/pages/Import/TaskList/index.module.less @@ -1,8 +1,9 @@ @import '~@app/common.less'; .nebulaDataImport { .taskBtns { - margin: 15px 0 20px; + padding: 15px 0 20px; display: flex; + border-top: 1px solid @gray; & > button:not(:last-child) { margin-right: 15px; } diff --git a/app/pages/Import/index.tsx b/app/pages/Import/index.tsx index af1b598a..dff8d673 100644 --- a/app/pages/Import/index.tsx +++ b/app/pages/Import/index.tsx @@ -4,7 +4,7 @@ import { Route, useHistory, useLocation } from 'react-router-dom'; import { trackPageView } from '@app/utils/stat'; import cls from 'classnames'; import { useI18n } from '@vesoft-inc/i18n'; -import FileList from './FileList'; +import DataSourceList from './DataSourceList'; import styles from './index.module.less'; import TaskList from './TaskList'; @@ -17,7 +17,7 @@ interface IProps { const Import = (props: IProps) => { const history = useHistory(); const location = useLocation(); - const [tab, setTab] = useState('files'); + const [tab, setTab] = useState('tasks'); const { intl } = useI18n(); const { showConfigDownload, showLogDownload, showTemplateModal } = props; useEffect(() => { @@ -25,7 +25,7 @@ const Import = (props: IProps) => { }, []); useEffect(() => { const path = location.pathname; - setTab(path.includes('files') ? 'files' : 'tasks'); + setTab(path.includes('dataSources') ? 'dataSources' : 'tasks'); }, [location.pathname]); const handleTabChange = (e: RadioChangeEvent) => { setTab(e.target.value); @@ -40,15 +40,15 @@ const Import = (props: IProps) => { buttonStyle="solid" onChange={handleTabChange} > - {intl.get('import.uploadFile')} {intl.get('import.importData')} + {intl.get('import.dataSourceManagement')}

{ icon: 'icon-studio-nav-import', title: intl.get('import.importData'), tip: intl.get('doc.importIntro'), - startLink: '/import/files', + startLink: '/import/tasks', docLink: intl.get('welcome.importModuleLink'), }, { From 024ab4e3b73632c76ac0a0b119a9d2d5b5ed8f6c Mon Sep 17 00:00:00 2001 From: hetao92 <18328704+hetao92@users.noreply.github.com> Date: Wed, 15 Mar 2023 03:58:23 +0000 Subject: [PATCH 02/11] feat: add multiple datasources --- app/config/locale/en-US.json | 25 ++- app/config/locale/zh-CN.json | 26 ++- app/config/service.ts | 26 +++ app/interfaces/datasource.ts | 13 ++ app/interfaces/import.ts | 19 +- .../DataSourceList/RemoteList/index.tsx | 102 ----------- .../PreviewFileModal/index.module.less | 0 .../LocalFileList/PreviewFileModal/index.tsx | 0 .../UploadConfigModal/index.module.less | 0 .../LocalFileList/UploadConfigModal/index.tsx | 0 .../LocalFileList/index.module.less | 0 .../LocalFileList/index.tsx | 2 +- .../DatasourceConfigModal/S3ConfigForm.tsx | 45 +++++ .../DatasourceConfigModal/SftpConfigForm.tsx | 33 ++++ .../DatasourceConfigModal/index.module.less | 22 +++ .../DatasourceConfigModal/index.tsx | 88 ++++++++++ .../RemoteList/index.module.less | 0 .../DatasourceList/RemoteList/index.tsx | 166 ++++++++++++++++++ .../index.module.less | 0 .../index.tsx | 12 +- .../Import/TaskList/TemplateModal/index.tsx | 2 +- app/pages/Import/index.tsx | 10 +- app/stores/datasource.ts | 66 +++++++ app/stores/index.ts | 3 +- .../datasourcebatchremovehandler.go | 33 ++++ .../datasource/datasourcelisthandler.go | 18 +- server/api/studio/internal/handler/routes.go | 5 + .../datasource/datasourcebatchremovelogic.go | 30 ++++ .../logic/datasource/datasourcelistlogic.go | 5 +- .../api/studio/internal/service/datasource.go | 123 ++++++++++--- server/api/studio/internal/types/types.go | 29 +-- server/api/studio/pkg/filestore/filestore.go | 3 +- server/api/studio/pkg/filestore/s3store.go | 37 +--- server/api/studio/pkg/filestore/sftpstore.go | 20 ++- server/api/studio/restapi/datasource.api | 34 ++-- 35 files changed, 771 insertions(+), 226 deletions(-) create mode 100644 app/interfaces/datasource.ts delete mode 100644 app/pages/Import/DataSourceList/RemoteList/index.tsx rename app/pages/Import/{DataSourceList => DatasourceList}/LocalFileList/PreviewFileModal/index.module.less (100%) rename app/pages/Import/{DataSourceList => DatasourceList}/LocalFileList/PreviewFileModal/index.tsx (100%) rename app/pages/Import/{DataSourceList => DatasourceList}/LocalFileList/UploadConfigModal/index.module.less (100%) rename app/pages/Import/{DataSourceList => DatasourceList}/LocalFileList/UploadConfigModal/index.tsx (100%) rename app/pages/Import/{DataSourceList => DatasourceList}/LocalFileList/index.module.less (100%) rename app/pages/Import/{DataSourceList => DatasourceList}/LocalFileList/index.tsx (99%) create mode 100644 app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/S3ConfigForm.tsx create mode 100644 app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/SftpConfigForm.tsx create mode 100644 app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.module.less create mode 100644 app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.tsx rename app/pages/Import/{DataSourceList => DatasourceList}/RemoteList/index.module.less (100%) create mode 100644 app/pages/Import/DatasourceList/RemoteList/index.tsx rename app/pages/Import/{DataSourceList => DatasourceList}/index.module.less (100%) rename app/pages/Import/{DataSourceList => DatasourceList}/index.tsx (75%) create mode 100644 app/stores/datasource.ts create mode 100644 server/api/studio/internal/handler/datasource/datasourcebatchremovehandler.go create mode 100644 server/api/studio/internal/logic/datasource/datasourcebatchremovelogic.go diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json index 68c1effc..818e490b 100644 --- a/app/config/locale/en-US.json +++ b/app/config/locale/en-US.json @@ -130,7 +130,15 @@ "maxBytes": "It cannot exceed {max} bytes", "ttlLimit": "The data type of the property must be int or timestamp", "associateNameRequired": "Associated {type} name is required", - "fileRequired": "Please select the file" + "fileRequired": "Please select the file", + "formHostRequired": "Host Required", + "formPortRequired": "Port Required", + "regionRequired": "Region Required", + "endpointRequired": "Endpoint Required", + "bucketRequired": "Bucket Required", + "accessKeyIdRequired": "Access Key ID Required", + "accessKeySecretRequired": "Access Key Secret Required", + "platformRequired": "Platform Required" }, "console": { "execTime": "Execution Time", @@ -253,17 +261,24 @@ "filePreview": "Preview file {name}", "uploadConfirm": "Upload Confirm", "localFiles": "Local files", - "cloudStorage": "Cloud storage", + "s3": "Cloud storage", "sftp": "SFTP", "newDataSource": "New Data Source", "deleteDataSource": "Delete Data Source", - "dataSourceList": "{type} list", + "datasourceList": "{type} list", "ipAddress": "IP Address:Port", "bucketName": "Bucket Name", "accessKeyId": "AccessKeyId", "region": "Region", - "addDate": "Add Date", - "account": "Account" + "createTime": "Add Date", + "endpoint": "Endpoint", + "accessKeySecret": "AccessKeySecret", + "dataSourceType": "Data Source Type", + "selectPlatform": "Select platform", + "enterAddress": "Enter endpoint", + "enterRegion": "Enter region", + "serverAddress": "Server Address", + "port": "Port" }, "schema": { "spaceList": "Graph Space List", diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json index 8d5b22c1..ffc4039b 100644 --- a/app/config/locale/zh-CN.json +++ b/app/config/locale/zh-CN.json @@ -130,7 +130,15 @@ "maxBytes": "不能超过 {max} 字节", "ttlLimit": "属性的数据类型必须是int或者timestamp", "associateNameRequired": "请选择关联 {type} 名称", - "fileRequired": "请选择文件" + "fileRequired": "请选择文件", + "formHostRequired": "请填写服务器IP地址", + "formPortRequired": "请填写服务器端口", + "regionRequired": "请选择区域", + "endpointRequired": "请填写 Endpoint", + "bucketRequired": "请填写 Bucket", + "accessKeyIdRequired": "请填写 Access Key ID", + "accessKeySecretRequired": "请填写 Access Key Secret", + "platformRequired": "请选择平台" }, "console": { "execTime": "执行时间消耗", @@ -253,17 +261,25 @@ "filePreview": "预览文件 {name}", "uploadConfirm": "上传文件确认", "localFiles": "本地文件", - "cloudStorage": "云存储", + "s3": "云存储", "sftp": "SFTP", "newDataSource": "新建数据源", "deleteDataSource": "删除数据源", - "dataSourceList": "{type}列表", + "datasourceList": "{type}列表", "ipAddress": "IP 地址：端口", "bucketName": "Bucket 名称", "accessKeyId": "AccessKeyId", "region": "区域", - "addDate": "添加日期", - "account": "账号" + "createTime": "添加日期", + "account": "账号", + "endpoint": "Endpoint", + "accessKeySecret": "AccessKeySecret", + "dataSourceType": "数据源类型", + "selectPlatform": "选择平台", + "enterAddress": "请输入终端节点地址", + "enterRegion": "请输入区域", + "serverAddress": "服务器地址", + "port": "端口" }, "schema": { "spaceList": "图空间列表", diff --git a/app/config/service.ts b/app/config/service.ts index dc966bb4..603e8945 100644 --- a/app/config/service.ts +++ b/app/config/service.ts @@ -90,6 +90,32 @@ const service = { saveFavorite: (params, config?) => { return post(`/api/favorites`)(params, config); }, + + // datasource + getDatasourceList: (params?, config?) => { + return get('/api/datasources')(params, config); + }, + addDatasource: (params, config?) => { + return post('/api/datasources')(params, config); + }, + updateDatasource: (params, config?) => { + const { id, ...restParams } = params; + return put(`/api/datasources/${id}`)(restParams, config); + }, + deleteDatasource: (id: number, config?) => { + return _delete(`/api/datasources/${id}`)(undefined, config); + }, + batchDeleteDatasource: (payload, config?) => { + return _delete(`/api/datasources`)(undefined, { data: payload }); + }, + getDatasourceDetail: (params, config?) => { + const { id, ...restParams } = params; + return get(`/api/datasources/${id}/contents`)(restParams, config); + }, + previewFile: (params, config?) => { + const { id, ...restParams } = params; + return get(`/api/datasources/${id}/file-preview`)(restParams, config); + }, }; export const updateService = (partService: any) => { diff --git a/app/interfaces/datasource.ts b/app/interfaces/datasource.ts new file mode 100644 index 00000000..3a1aa151 --- /dev/null +++ b/app/interfaces/datasource.ts @@ -0,0 +1,13 @@ +export enum IRemoteType { + 'S3' = 's3', + 'Sftp' = 'sftp', +} + + +export interface ICloudStorage { + ipAddress: string; + bucketName: string; + accessKeyId: string; + region: string; + createTime: number; +} \ No newline at end of file diff --git a/app/interfaces/import.ts b/app/interfaces/import.ts index b17114f5..37926f15 100644 --- a/app/interfaces/import.ts +++ b/app/interfaces/import.ts @@ -18,8 +18,10 @@ export interface ITaskStats { totalRequest: number; totalLatency: number; totalRespTime: number; - failedProcessed: number; // The number of nodes and edges that have failed to be processed. - totalProcessed: number; // The number of nodes and edges that have been processed. + /* The number of nodes and edges that have failed to be processed. */ + failedProcessed: number; + /* 123. */ + totalProcessed: number; } export interface ITaskItem { id: number; @@ -69,17 +71,4 @@ export interface StudioFile extends RcFile { delimiter?: string; sample?: string; content?: any[]; -} - -export enum IRemoteType { - 'Cloud' = 'cloudStorage', - 'Sftp' = 'sftp', -} - -export interface ICloudStorage { - ipAddress: string; - bucketName: string; - accessKeyId: string; - region: string; - addDate: number; } \ No newline at end of file diff --git a/app/pages/Import/DataSourceList/RemoteList/index.tsx b/app/pages/Import/DataSourceList/RemoteList/index.tsx deleted file mode 100644 index 447975ff..00000000 --- a/app/pages/Import/DataSourceList/RemoteList/index.tsx +++ /dev/null @@ -1,102 +0,0 @@ -import React, { useCallback, useEffect, useState } from 'react'; -import { observer } from 'mobx-react-lite'; -import { useStore } from '@app/stores'; -import { trackPageView } from '@app/utils/stat'; -import { Button, Popconfirm, Table } from 'antd'; -import Icon from '@app/components/Icon'; -import cls from 'classnames'; -import { IRemoteType } from '@app/interfaces/import'; -import { useI18n } from '@vesoft-inc/i18n'; - -import styles from './index.module.less'; - -interface IProps { - type: IRemoteType; -} -const cloudKeys = ['ipAddress', 'bucketName', 'accessKeyId', 'region', 'addDate']; -const sftpKeys = ['ipAddress', 'account', 'addDate']; -const columnKeys = { - [IRemoteType.Cloud]: cloudKeys, - [IRemoteType.Sftp]: sftpKeys, -}; -const FileList = (props: IProps) => { - const { type } = props; - const { files, global } = useStore(); - const { intl } = useI18n(); - const { fileList, deleteFile, getFiles } = files; - const [loading, setLoading] = useState(false); - const [selectItems, setSelectItems] = useState([]); - const columns = columnKeys[type].map(key => ({ - title: intl.get(`import.${key}`), - dataIndex: key, - })).concat(({ - title: intl.get('common.operation'), - key: 'operation', - render: (_, file) => (
- - deleteFile([file.name])} - title={intl.get('common.ask')} - okText={intl.get('common.confirm')} - cancelText={intl.get('common.cancel')} - > - - -
) - } as any)); - - - const getFileList = async () => { - !loading && setLoading(true); - await getFiles(); - setLoading(false); - }; - const handleDeleteFiles = useCallback(async () => { - const flag = await deleteFile(selectItems); - flag && setSelectItems([]); - }, [selectItems]); - useEffect(() => { - getFileList(); - trackPageView('/import/dataSources'); - }, []); - return ( -
-
- - - - -
-
-

{intl.get('import.dataSourceList', { type: intl.get(`import.${type}`) })} ({fileList.length})

-
setSelectItems(selectedRowKeys as string[]), - }} - columns={columns} - rowKey="name" - pagination={false} - /> - - - ); -}; - -export default observer(FileList); diff --git a/app/pages/Import/DataSourceList/LocalFileList/PreviewFileModal/index.module.less b/app/pages/Import/DatasourceList/LocalFileList/PreviewFileModal/index.module.less similarity index 100% rename from app/pages/Import/DataSourceList/LocalFileList/PreviewFileModal/index.module.less rename to app/pages/Import/DatasourceList/LocalFileList/PreviewFileModal/index.module.less diff --git a/app/pages/Import/DataSourceList/LocalFileList/PreviewFileModal/index.tsx b/app/pages/Import/DatasourceList/LocalFileList/PreviewFileModal/index.tsx similarity index 100% rename from app/pages/Import/DataSourceList/LocalFileList/PreviewFileModal/index.tsx rename to app/pages/Import/DatasourceList/LocalFileList/PreviewFileModal/index.tsx diff --git a/app/pages/Import/DataSourceList/LocalFileList/UploadConfigModal/index.module.less b/app/pages/Import/DatasourceList/LocalFileList/UploadConfigModal/index.module.less similarity index 100% rename from app/pages/Import/DataSourceList/LocalFileList/UploadConfigModal/index.module.less rename to app/pages/Import/DatasourceList/LocalFileList/UploadConfigModal/index.module.less diff --git a/app/pages/Import/DataSourceList/LocalFileList/UploadConfigModal/index.tsx b/app/pages/Import/DatasourceList/LocalFileList/UploadConfigModal/index.tsx similarity index 100% rename from app/pages/Import/DataSourceList/LocalFileList/UploadConfigModal/index.tsx rename to app/pages/Import/DatasourceList/LocalFileList/UploadConfigModal/index.tsx diff --git a/app/pages/Import/DataSourceList/LocalFileList/index.module.less b/app/pages/Import/DatasourceList/LocalFileList/index.module.less similarity index 100% rename from app/pages/Import/DataSourceList/LocalFileList/index.module.less rename to app/pages/Import/DatasourceList/LocalFileList/index.module.less diff --git a/app/pages/Import/DataSourceList/LocalFileList/index.tsx b/app/pages/Import/DatasourceList/LocalFileList/index.tsx similarity index 99% rename from app/pages/Import/DataSourceList/LocalFileList/index.tsx rename to app/pages/Import/DatasourceList/LocalFileList/index.tsx index 62ed2819..e6ecd853 100644 --- a/app/pages/Import/DataSourceList/LocalFileList/index.tsx +++ b/app/pages/Import/DatasourceList/LocalFileList/index.tsx @@ -102,7 +102,7 @@ const FileList = () => { }, [selectFiles]); useEffect(() => { getFileList(); - trackPageView('/import/dataSources'); + trackPageView('/import/datasources'); }, []); return (
diff --git a/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/S3ConfigForm.tsx b/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/S3ConfigForm.tsx new file mode 100644 index 00000000..8998f811 --- /dev/null +++ b/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/S3ConfigForm.tsx @@ -0,0 +1,45 @@ +import { useI18n } from '@vesoft-inc/i18n'; +import { Input, Form, Select, FormInstance } from 'antd'; +import React from 'react'; +import { observer } from 'mobx-react-lite'; +import styles from './index.module.less'; + +const FormItem = Form.Item; +interface IProps { + formRef: FormInstance +} + +const S3ConfigForm = (props: IProps) => { + const { formRef } = props; + const { intl } = useI18n(); + + return ( + + + + + + + + + + + + + + + + + + + + + + + ); +}; + +export default observer(S3ConfigForm); diff --git a/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/SftpConfigForm.tsx b/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/SftpConfigForm.tsx new file mode 100644 index 00000000..7883c164 --- /dev/null +++ b/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/SftpConfigForm.tsx @@ -0,0 +1,33 @@ +import { useI18n } from '@vesoft-inc/i18n'; +import { Input, Form, FormInstance } from 'antd'; +import React from 'react'; +import { observer } from 'mobx-react-lite'; + +const FormItem = Form.Item; +interface IProps { + formRef: FormInstance +} + +const SftpConfigForm = (props: IProps) => { + const { formRef } = props; + const { intl } = useI18n(); + + return ( + + + + + + + + + + + + + + + ); +}; + +export default observer(SftpConfigForm); diff --git a/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.module.less b/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.module.less new file mode 100644 index 00000000..3f82cacd --- /dev/null +++ b/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.module.less @@ -0,0 +1,22 @@ +@import '~@app/common.less'; + +.dataSourceModal { + :global { + .ant-modal-header { + border-bottom: none; + } + .ant-modal-body { + padding: 0 10px 35px; + } + } + .mixedItem { + margin-bottom: 0; + } + .btns { + text-align: center; + :global(.ant-btn) { + width: 180px; + } + } +} + diff --git a/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.tsx b/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.tsx new file mode 100644 index 00000000..1b9e8fe1 --- /dev/null +++ b/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.tsx @@ -0,0 +1,88 @@ +import { useI18n } from '@vesoft-inc/i18n'; +import { Button, Modal, Form, Select, message } from 'antd'; +import React, { useState } from 'react'; +import { IRemoteType } from '@app/interfaces/datasource'; +import { useStore } from '@app/stores'; +import { observer } from 'mobx-react-lite'; +import S3ConfigForm from './S3ConfigForm'; +import SftpConfigForm from './SftpConfigForm'; +import styles from './index.module.less'; + +const FormItem = Form.Item; +interface IProps { + visible: boolean; + onConfirm: () => void; + onCancel: () => void; + type?: IRemoteType; + data?: any; +} + +const fomrItemLayout = { + labelCol: { span: 7, offset: 2 }, + wrapperCol: { span: 10 }, +}; + +const DatasourceConfigModal = (props: IProps) => { + const { visible, type, onCancel, onConfirm, data } = props; + const { datasource } = useStore(); + const { addDataSource } = datasource; + const { intl } = useI18n(); + const [form] = Form.useForm(); + const [loading, setLoading] = useState(false); + + const submit = async (values: any) => { + const { platform, ...rest } = values; + setLoading(true); + const flag = await addDataSource({ + type: values.type, + name: '', + ...rest + }); + setLoading(false); + flag && (message.success(intl.get('schema.createSuccess')), onConfirm()); + }; + console.log('data', data); + + return ( + +
+ {!type && + + } + + {({ getFieldValue }) => { + const configType = type || getFieldValue('type'); + switch (configType) { + case IRemoteType.S3: + return ; + case IRemoteType.Sftp: + return ; + default: + return null; + } + }} + +
+ +
+ + + ); +}; + +export default observer(DatasourceConfigModal); diff --git a/app/pages/Import/DataSourceList/RemoteList/index.module.less b/app/pages/Import/DatasourceList/RemoteList/index.module.less similarity index 100% rename from app/pages/Import/DataSourceList/RemoteList/index.module.less rename to app/pages/Import/DatasourceList/RemoteList/index.module.less diff --git a/app/pages/Import/DatasourceList/RemoteList/index.tsx b/app/pages/Import/DatasourceList/RemoteList/index.tsx new file mode 100644 index 00000000..88eaf939 --- /dev/null +++ b/app/pages/Import/DatasourceList/RemoteList/index.tsx @@ -0,0 +1,166 @@ +import React, { useCallback, useEffect, useState } from 'react'; +import { observer } from 'mobx-react-lite'; +import { useStore } from '@app/stores'; +import { trackPageView } from '@app/utils/stat'; +import { Button, message, Popconfirm, Table } from 'antd'; +import Icon from '@app/components/Icon'; +import cls from 'classnames'; +import { IRemoteType } from '@app/interfaces/datasource'; +import { useI18n } from '@vesoft-inc/i18n'; + +import dayjs from 'dayjs'; +import DatasourceConfigModal from './DatasourceConfigModal'; +import styles from './index.module.less'; + +interface IProps { + type: IRemoteType; +} +const cloudKeys = [ + { + title: 'ipAddress', + key: ['s3Config', 'endpoint'] + }, + { + title: 'bucketName', + key: ['s3Config', 'bucket'] + }, + { + title: 'accessKeyId', + key: ['s3Config', 'accessKey'] + }, + { + title: 'region', + key: ['s3Config', 'region'] + }, + { + title: 'createTime', + key: 'createTime', + render: data => data && dayjs(data).format('YYYY-MM-DD HH:mm:ss') + }, +]; +const sftpKeys = [{ + title: 'ipAddress', + key: ['sftpConfig', 'host'], + render: (_, row) => `${row.sftpConfig.host}:${row.sftpConfig.port}` +}, +{ + title: 'account', + key: ['sftpConfig', 'username'] +}, +{ + title: 'createTime', + key: 'createTime', + render: data => data && dayjs(data).format('YYYY-MM-DD HH:mm:ss') +} +]; +const columnKeys = { + [IRemoteType.S3]: cloudKeys, + [IRemoteType.Sftp]: sftpKeys, +}; + +const DatasourceList = (props: IProps) => { + const { type } = props; + const { datasource } = useStore(); + const { intl } = useI18n(); + const { getDatasourceList, datasourceList, getDatasourceDetail, previewFile, deleteDataSource, batchDeleteDatasource } = datasource; + const [data, setData] = useState([]); + const [editData, setEditData] = useState(null); + const [loading, setLoading] = useState(false); + const [visible, setVisible] = useState(false); + const [selectItems, setSelectItems] = useState([]); + + const editItem = (item) => { + setEditData(item); + setVisible(true); + }; + const deleteItem = useCallback(async id => { + const flag = await deleteDataSource(id); + flag && (getList(), message.success(intl.get('common.deleteSuccess'))); + }, []); + const columns = columnKeys[type].map(item => ({ + title: intl.get(`import.${item.title}`), + dataIndex: item.key, + render: item.render + })).concat(({ + title: intl.get('common.operation'), + key: 'operation', + render: (_, item) => (
+ + deleteItem(item.id)} + title={intl.get('common.ask')} + okText={intl.get('common.confirm')} + cancelText={intl.get('common.cancel')} + > + + +
) + } as any)); + + + const getList = async () => { + !loading && setLoading(true); + const data = await getDatasourceList({ type }); + setData(data); + setLoading(false); + }; + const handleDeleteDatasource = useCallback(async () => { + const flag = await batchDeleteDatasource(selectItems); + if (!flag) return; + message.success(intl.get('common.deleteSuccess')); + getList(); + setSelectItems([]); + }, [selectItems]); + + const handleRefresh = () => { + getList(); + setVisible(false); + }; + + + useEffect(() => { + getList(); + trackPageView('/import/datasources'); + }, []); + return ( +
+
+ + + + +
+
+

{intl.get('import.datasourceList', { type: intl.get(`import.${type}`) })} ({datasourceList.length})

+
setSelectItems(selectedRowKeys as number[]), + }} + columns={columns} + rowKey="id" + pagination={false} + /> + + {visible && setVisible(false)} onConfirm={handleRefresh} />} + + ); +}; + +export default observer(DatasourceList); diff --git a/app/pages/Import/DataSourceList/index.module.less b/app/pages/Import/DatasourceList/index.module.less similarity index 100% rename from app/pages/Import/DataSourceList/index.module.less rename to app/pages/Import/DatasourceList/index.module.less diff --git a/app/pages/Import/DataSourceList/index.tsx b/app/pages/Import/DatasourceList/index.tsx similarity index 75% rename from app/pages/Import/DataSourceList/index.tsx rename to app/pages/Import/DatasourceList/index.tsx index 763f8915..66ce68a6 100644 --- a/app/pages/Import/DataSourceList/index.tsx +++ b/app/pages/Import/DatasourceList/index.tsx @@ -2,12 +2,12 @@ import React from 'react'; import { observer } from 'mobx-react-lite'; import { Tabs, TabsProps } from 'antd'; import { useI18n } from '@vesoft-inc/i18n'; -import { IRemoteType } from '@app/interfaces/import'; +import { IRemoteType } from '@app/interfaces/datasource'; import LocalFileList from './LocalFileList'; import RemoteList from './RemoteList'; import styles from './index.module.less'; -const DataSourceList = () => { +const DatasourceList = () => { const { intl } = useI18n(); const items: TabsProps['items'] = [ { @@ -16,9 +16,9 @@ const DataSourceList = () => { children: }, { - key: 'cloud', - label: intl.get('import.cloudStorage'), - children: + key: 's3', + label: intl.get('import.s3'), + children: }, { key: 'sftp', @@ -33,4 +33,4 @@ const DataSourceList = () => { ); }; -export default observer(DataSourceList); +export default observer(DatasourceList); diff --git a/app/pages/Import/TaskList/TemplateModal/index.tsx b/app/pages/Import/TaskList/TemplateModal/index.tsx index 68f3627c..7180f158 100644 --- a/app/pages/Import/TaskList/TemplateModal/index.tsx +++ b/app/pages/Import/TaskList/TemplateModal/index.tsx @@ -118,7 +118,7 @@ const TemplateModal = (props: IProps) => { {!config ?

{intl.get('import.fileUploadRequired')} - {intl.get('import.uploadFile')} + {intl.get('import.uploadFile')} {intl.get('import.fileUploadRequired2')}

diff --git a/app/pages/Import/index.tsx b/app/pages/Import/index.tsx index dff8d673..3905aafa 100644 --- a/app/pages/Import/index.tsx +++ b/app/pages/Import/index.tsx @@ -4,7 +4,7 @@ import { Route, useHistory, useLocation } from 'react-router-dom'; import { trackPageView } from '@app/utils/stat'; import cls from 'classnames'; import { useI18n } from '@vesoft-inc/i18n'; -import DataSourceList from './DataSourceList'; +import DatasourceList from './DatasourceList'; import styles from './index.module.less'; import TaskList from './TaskList'; @@ -25,7 +25,7 @@ const Import = (props: IProps) => { }, []); useEffect(() => { const path = location.pathname; - setTab(path.includes('dataSources') ? 'dataSources' : 'tasks'); + setTab(path.includes('datasources') ? 'datasources' : 'tasks'); }, [location.pathname]); const handleTabChange = (e: RadioChangeEvent) => { setTab(e.target.value); @@ -41,14 +41,14 @@ const Import = (props: IProps) => { onChange={handleTabChange} > {intl.get('import.importData')} - {intl.get('import.dataSourceManagement')} + {intl.get('import.dataSourceManagement')}

) => { + Object.keys(payload).forEach(key => Object.prototype.hasOwnProperty.call(this, key) && (this[key] = payload[key])); + }; + + addDataSource = async (payload) => { + const { code } = await service.addDatasource(payload); + return code === 0; + }; + getDatasourceList = async (payload?: { type?: string }) => { + const { code, data } = await service.getDatasourceList(payload); + if(code === 0) { + return data.list; + } + }; + deleteDataSource = async (id: number) => { + const { code } = await service.deleteDatasource(id); + return code === 0; + }; + batchDeleteDatasource = async (ids: number[]) => { + const { code } = await service.batchDeleteDatasource({ ids }); + return code === 0; + }; + getDatasourceDetail = async (payload: { + id: string, + path?: string, + }) => { + const { id } = payload; + const { code, data } = await service.getDatasourceDetail({ id }); + if(code === 0) { + console.log('data', data); + return data; + } + }; + previewFile = async (payload: { + id: string, + path?: string, + }) => { + const { id } = payload; + const { code, data } = await service.previewFile({ id, path: 'importer-hetao-test/player.csv' }); + if(code === 0) { + console.log('data', data); + return data; + } + }; +} + +const datasourceStore = new DatasourceStore(); + +export default datasourceStore; + diff --git a/app/stores/index.ts b/app/stores/index.ts index fdaa6b15..fb1628ea 100644 --- a/app/stores/index.ts +++ b/app/stores/index.ts @@ -7,8 +7,9 @@ import schema from './schema'; import graphInstances from './graphInstances'; import sketchModel from './sketchModel'; import welcome from './welcome'; +import datasource from './datasource'; -const rootStore = { global, files, console, dataImport, schema, graphInstances, sketchModel, welcome }; +const rootStore = { global, files, console, dataImport, schema, graphInstances, sketchModel, welcome, datasource }; const rootStoreRef = { current: rootStore }; // @ts-ignore window.studioStore = rootStore; diff --git a/server/api/studio/internal/handler/datasource/datasourcebatchremovehandler.go b/server/api/studio/internal/handler/datasource/datasourcebatchremovehandler.go new file mode 100644 index 00000000..58126b9f --- /dev/null +++ b/server/api/studio/internal/handler/datasource/datasourcebatchremovehandler.go @@ -0,0 +1,33 @@ +// Code generated by goctl. DO NOT EDIT. +package datasource + +import ( + "net/http" + + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/datasource" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" +) + +func DatasourceBatchRemoveHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.DatasourceBatchRemoveRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithErrorMessage(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + + l := datasource.NewDatasourceBatchRemoveLogic(r.Context(), svcCtx) + err := l.DatasourceBatchRemove(req) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + } +} diff --git a/server/api/studio/internal/handler/datasource/datasourcelisthandler.go b/server/api/studio/internal/handler/datasource/datasourcelisthandler.go index 36bc74fc..e08518ea 100644 --- a/server/api/studio/internal/handler/datasource/datasourcelisthandler.go +++ b/server/api/studio/internal/handler/datasource/datasourcelisthandler.go @@ -4,14 +4,30 @@ package datasource import ( "net/http" + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/datasource" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" ) func DatasourceListHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + var req types.DatasourceListRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithErrorMessage(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + l := datasource.NewDatasourceListLogic(r.Context(), svcCtx) - data, err := l.DatasourceList() + data, err := l.DatasourceList(req) svcCtx.ResponseHandler.Handle(w, r, data, err) } } diff --git a/server/api/studio/internal/handler/routes.go b/server/api/studio/internal/handler/routes.go index b281590b..cdd01ed1 100644 --- a/server/api/studio/internal/handler/routes.go +++ b/server/api/studio/internal/handler/routes.go @@ -217,6 +217,11 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { Path: "/api/datasources/:id", Handler: datasource.DatasourceRemoveHandler(serverCtx), }, + { + Method: http.MethodDelete, + Path: "/api/datasources", + Handler: datasource.DatasourceBatchRemoveHandler(serverCtx), + }, { Method: http.MethodGet, Path: "/api/datasources", diff --git a/server/api/studio/internal/logic/datasource/datasourcebatchremovelogic.go b/server/api/studio/internal/logic/datasource/datasourcebatchremovelogic.go new file mode 100644 index 00000000..e25cb633 --- /dev/null +++ b/server/api/studio/internal/logic/datasource/datasourcebatchremovelogic.go @@ -0,0 +1,30 @@ +package datasource + +import ( + "context" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type DatasourceBatchRemoveLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewDatasourceBatchRemoveLogic(ctx context.Context, svcCtx *svc.ServiceContext) DatasourceBatchRemoveLogic { + return DatasourceBatchRemoveLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *DatasourceBatchRemoveLogic) DatasourceBatchRemove(req types.DatasourceBatchRemoveRequest) error { + return service.NewDatasourceService(l.ctx, l.svcCtx).BatchRemove(req) +} diff --git a/server/api/studio/internal/logic/datasource/datasourcelistlogic.go b/server/api/studio/internal/logic/datasource/datasourcelistlogic.go index ebefd0e4..05c57c5a 100644 --- a/server/api/studio/internal/logic/datasource/datasourcelistlogic.go +++ b/server/api/studio/internal/logic/datasource/datasourcelistlogic.go @@ -2,6 +2,7 @@ package datasource import ( "context" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" @@ -24,6 +25,6 @@ func NewDatasourceListLogic(ctx context.Context, svcCtx *svc.ServiceContext) Dat } } -func (l *DatasourceListLogic) DatasourceList() (resp *types.DatasourceData, err error) { - return service.NewDatasourceService(l.ctx, l.svcCtx).List() +func (l *DatasourceListLogic) DatasourceList(req types.DatasourceListRequest) (resp *types.DatasourceData, err error) { + return service.NewDatasourceService(l.ctx, l.svcCtx).List(req) } diff --git a/server/api/studio/internal/service/datasource.go b/server/api/studio/internal/service/datasource.go index 875d9f76..b8b83968 100644 --- a/server/api/studio/internal/service/datasource.go +++ b/server/api/studio/internal/service/datasource.go @@ -4,7 +4,12 @@ import ( "context" "encoding/json" "fmt" + "net/url" + "strconv" + "strings" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" @@ -18,14 +23,14 @@ import ( "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/utils" "github.com/zeromicro/go-zero/core/logx" "golang.org/x/crypto/ssh" - "strconv" ) type ( DatasourceService interface { Add(request types.DatasourceAddRequest) (*types.DatasourceAddData, error) - List() (*types.DatasourceData, error) + List(request types.DatasourceListRequest) (*types.DatasourceData, error) Remove(request types.DatasourceRemoveRequest) error + BatchRemove(request types.DatasourceBatchRemoveRequest) error ListContents(request types.DatasourceListContentsRequest) (*types.DatasourceListContentsData, error) PreviewFile(request types.DatasourcePreviewFileRequest) (*types.DatasourcePreviewFileData, error) } @@ -54,14 +59,14 @@ func (d *datasourceService) Add(request types.DatasourceAddRequest) (*types.Data switch request.Type { case "s3": c := request.S3Config - if err := d.testConnectionS3(c.Endpoint, c.Region, c.AccessKey, c.AccessSecret); err != nil { + if err := d.testConnectionS3(c.Endpoint, c.Region, c.Bucket, c.AccessKey, c.AccessSecret); err != nil { return nil, err } secret := c.AccessSecret c.AccessSecret = "" cstr, err := json.Marshal(c) if err != nil { - return nil, ecode.WithBadRequest(err, "json stringify error") + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err, "json stringify error") } crypto, err := utils.Encrypt([]byte(secret), []byte(cipher)) id, err := d.save(request.Type, request.Name, string(cstr), crypto) @@ -71,7 +76,6 @@ func (d *datasourceService) Add(request types.DatasourceAddRequest) (*types.Data return &types.DatasourceAddData{ ID: id, }, nil - break case "sftp": c := request.SFTPConfig if err := d.testConnectionSFTP(c.Host, c.Port, c.Username, c.Password); err != nil { @@ -82,7 +86,7 @@ func (d *datasourceService) Add(request types.DatasourceAddRequest) (*types.Data cstr, err := json.Marshal(c) if err != nil { d.Logger.Errorf("json stringify error", c) - return nil, ecode.WithBadRequest(err, "json stringify error") + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err, "json stringify error") } crypto, err := utils.Encrypt([]byte(pwd), []byte(cipher)) id, err := d.save(request.Type, request.Name, string(cstr), crypto) @@ -92,19 +96,20 @@ func (d *datasourceService) Add(request types.DatasourceAddRequest) (*types.Data return &types.DatasourceAddData{ ID: id, }, nil - break } return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, nil, "datasource type can't support'") } -func (d *datasourceService) List() (*types.DatasourceData, error) { +func (d *datasourceService) List(request types.DatasourceListRequest) (*types.DatasourceData, error) { user := d.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) host := user.Address + ":" + strconv.Itoa(user.Port) var dbsList []db.Datasource result := db.CtxDB.Where("host = ?", host). - Where("username = ?", user.Username). - Order("create_time desc"). - Find(&dbsList) + Where("username = ?", user.Username) + if request.Type != "" { + result = result.Where("type = ?", request.Type) + } + result = result.Order("create_time desc").Find(&dbsList) if result.Error != nil { return nil, d.gormErrorWrapper(result.Error) } @@ -118,19 +123,17 @@ func (d *datasourceService) List() (*types.DatasourceData, error) { } switch config.Type { case "s3": - config.S3Config = types.DatasourceS3Config{} + config.S3Config = &types.DatasourceS3Config{} jsonConfig := item.Config if err := json.Unmarshal([]byte(jsonConfig), &config.S3Config); err != nil { return nil, ecode.WithInternalServer(err, "parse json failed") } - break case "sftp": - config.SFTPConfig = types.DatasourceSFTPConfig{} + config.SFTPConfig = &types.DatasourceSFTPConfig{} jsonConfig := item.Config if err := json.Unmarshal([]byte(jsonConfig), &config.SFTPConfig); err != nil { return nil, ecode.WithInternalServer(err, "parse json failed") } - break } items = append(items, config) } @@ -152,7 +155,22 @@ func (d *datasourceService) Remove(request types.DatasourceRemoveRequest) error } if result.RowsAffected == 0 { - return ecode.WithBadRequest(fmt.Errorf("test"), "there is available item to delete") + return ecode.WithErrorMessage(ecode.ErrBadRequest, fmt.Errorf("test"), "there is available item to delete") + } + + return nil +} + +func (d *datasourceService) BatchRemove(request types.DatasourceBatchRemoveRequest) error { + user := d.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) + result := db.CtxDB.Where("id IN (?) AND username = ?", request.IDs, user.Username).Delete(&db.Datasource{}) + + if result.Error != nil { + return d.gormErrorWrapper(result.Error) + } + + if result.RowsAffected == 0 { + return ecode.WithErrorMessage(ecode.ErrBadRequest, fmt.Errorf("test"), "there is available item to delete") } return nil @@ -170,7 +188,7 @@ func (d *datasourceService) ListContents(request types.DatasourceListContentsReq } fileList, err := store.ListFiles(request.Path) if err != nil { - return nil, ecode.WithBadRequest(err, "listFiles failed") + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err, "listFiles failed") } return &types.DatasourceListContentsData{ List: fileList, @@ -189,7 +207,7 @@ func (d *datasourceService) PreviewFile(request types.DatasourcePreviewFileReque // read three lines contents, err := store.ReadFile(request.Path, 0, 4) if err != nil { - return nil, ecode.WithBadRequest(err, "readFiles failed") + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err, "readFiles failed") } return &types.DatasourcePreviewFileData{ @@ -205,7 +223,7 @@ func (d *datasourceService) findOne(datasourceId int) (*db.Datasource, error) { return nil, d.gormErrorWrapper(result.Error) } if result.RowsAffected == 0 { - return nil, ecode.WithBadRequest(nil, "datasource don't exist") + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, nil, "datasource don't exist") } secret, err := utils.Decrypt(dbs.Secret, []byte(cipher)) @@ -250,7 +268,52 @@ func (d *datasourceService) getFileStore(dbs *db.Datasource) (filestore.FileStor return store, nil } -func (d *datasourceService) testConnectionS3(endpoint, region, key, secret string) error { +func parseEndpoint(rawEndpoint string) (string, string, error) { + // endpointURL := "https://s3..amazonaws.com" + // endpointURL := "https://my-bucket.s3..amazonaws.com" + // endpointURL := "https://s3..amazonaws.com/my-bucket" + if !strings.HasPrefix(rawEndpoint, "https://") && !strings.HasPrefix(rawEndpoint, "http://") { + rawEndpoint = fmt.Sprintf("https://%s", rawEndpoint) + } + u, err := url.Parse(rawEndpoint) + if err != nil { + return "", "", err + } + host := u.Hostname() + parts := strings.SplitN(host, ".", 2) + var ( + bucket string + endpoint string + ) + if parts[0] == "s3" { + // Format: https://s3..amazonaws.com + endpoint = u.Host + if u.Path != "" { + pathParts := strings.SplitN(u.Path, "/", 3) + bucket = pathParts[1] + } + } else { + // Format: https://.s3..amazonaws.com or https://s3.amazonaws.com/ + if parts[0] == "s3.amazonaws" { + // Format: https://s3.amazonaws.com/ + endpoint = fmt.Sprintf("https://%s", u.Host) + if u.Path != "" { + pathParts := strings.SplitN(u.Path, "/", 3) + bucket = pathParts[1] + } + } else { + // Format: https://.s3..amazonaws.com + bucket = parts[0] + endpoint = fmt.Sprintf("https://%s", parts[1]) + } + } + return endpoint, bucket, nil +} +func (d *datasourceService) testConnectionS3(endpoint, region, bucket, key, secret string) error { + endpoint, parsedBucket, err := parseEndpoint(endpoint) + if err != nil { + return ecode.WithErrorMessage(ecode.ErrBadRequest, err) + } sess, err := session.NewSession(&aws.Config{ Region: aws.String(region), Endpoint: aws.String(endpoint), @@ -261,9 +324,19 @@ func (d *datasourceService) testConnectionS3(endpoint, region, key, secret strin } svc := s3.New(sess) - _, err = svc.ListBuckets(nil) + if bucket != parsedBucket { + return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "The bucket name does not match the bucket in the endpoint") + } + _, err = svc.HeadBucket(&s3.HeadBucketInput{ + Bucket: aws.String(bucket), + }) if err != nil { - return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "Failed to list buckets") + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "NotFound" { + return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "Bucket does not exist") + } + } + return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "Failed to head bucket") } return nil @@ -282,21 +355,21 @@ func (d *datasourceService) testConnectionSFTP(host string, port int, username s // connect to the remote SSH server conn, err := ssh.Dial("tcp", fmt.Sprintf("%s:%d", host, port), config) if err != nil { - return ecode.WithBadRequest(err, "failed to dial SSH server") + return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "failed to dial SSH server") } defer conn.Close() // create an SFTP client session client, err := sftp.NewClient(conn) if err != nil { - return ecode.WithBadRequest(err, "failed to create SFTP session") + return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "failed to create SFTP session") } defer client.Close() // test the SFTP connection by listing the remote directory _, err = client.ReadDir("/") if err != nil { - return ecode.WithBadRequest(err, "failed to list remote directory") + return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "failed to list remote directory") } return nil diff --git a/server/api/studio/internal/types/types.go b/server/api/studio/internal/types/types.go index a52f1e2a..9683dc31 100644 --- a/server/api/studio/internal/types/types.go +++ b/server/api/studio/internal/types/types.go @@ -344,6 +344,7 @@ type FavoriteIDResult struct { type DatasourceS3Config struct { Endpoint string `json:"endpoint"` Region string `json:"region"` + Bucket string `json:"bucket"` AccessKey string `json:"accessKey"` AccessSecret string `json:"accessSecret"` } @@ -356,27 +357,35 @@ type DatasourceSFTPConfig struct { } type DatasourceAddRequest struct { - Type string `json:"type"` - Name string `json:"name"` - S3Config DatasourceS3Config `json:"s3Config,optional"` - SFTPConfig DatasourceSFTPConfig `json:"sftpConfig,optional"` + Type string `json:"type"` + Name string `json:"name"` + S3Config *DatasourceS3Config `json:"s3Config,optional"` + SFTPConfig *DatasourceSFTPConfig `json:"sftpConfig,optional"` } type DatasourceAddData struct { ID int `json:"id"` } +type DatasourceListRequest struct { + Type string `form:"type,optional"` +} + type DatasourceRemoveRequest struct { ID int `path:"id"` } +type DatasourceBatchRemoveRequest struct { + IDs []int `json:"ids"` +} + type DatasourceConfig struct { - ID int `json:"id"` - Type string `json:"type"` - Name string `json:"name"` - S3Config DatasourceS3Config `json:"s3Config,optional"` - SFTPConfig DatasourceSFTPConfig `json:"sftpConfig,optional"` - CreateTime int64 `json:"createTime,optional"` + ID int `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + S3Config *DatasourceS3Config `json:"s3Config,optional"` + SFTPConfig *DatasourceSFTPConfig `json:"sftpConfig,optional"` + CreateTime int64 `json:"createTime,optional"` } type DatasourceListContentsRequest struct { diff --git a/server/api/studio/pkg/filestore/filestore.go b/server/api/studio/pkg/filestore/filestore.go index ce6c66e9..b1ed5975 100644 --- a/server/api/studio/pkg/filestore/filestore.go +++ b/server/api/studio/pkg/filestore/filestore.go @@ -21,6 +21,7 @@ type ( S3Config struct { Endpoint string Region string + Bucket string AccessKey string AccessSecret string } @@ -33,7 +34,7 @@ func NewFileStore(typ, config, secret string) (FileStore, error) { if err := json.Unmarshal([]byte(config), &c); err != nil { return nil, errors.New("parse the s3 config error") } - return NewS3Store(c.Endpoint, c.Region, c.AccessKey, secret) + return NewS3Store(c.Endpoint, c.Region, c.Bucket, c.AccessKey, secret) case "sftp": var c SftpConfig if err := json.Unmarshal([]byte(config), &c); err != nil { diff --git a/server/api/studio/pkg/filestore/s3store.go b/server/api/studio/pkg/filestore/s3store.go index ff7944eb..073ce6d0 100644 --- a/server/api/studio/pkg/filestore/s3store.go +++ b/server/api/studio/pkg/filestore/s3store.go @@ -3,9 +3,9 @@ package filestore import ( "bufio" "errors" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" - "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" @@ -14,12 +14,12 @@ import ( type S3Store struct { S3Client s3iface.S3API + Bucket string } -func NewS3Store(endpoint, region, accessKey, accessSecret string) (*S3Store, error) { +func NewS3Store(endpoint, region, bucket, accessKey, accessSecret string) (*S3Store, error) { sess, err := session.NewSession(&aws.Config{ Region: aws.String(region), - Endpoint: aws.String(endpoint), Credentials: credentials.NewStaticCredentials(accessKey, accessSecret, ""), }) if err != nil { @@ -29,16 +29,13 @@ func NewS3Store(endpoint, region, accessKey, accessSecret string) (*S3Store, err svc := s3.New(sess) return &S3Store{ S3Client: svc, + Bucket: bucket, }, nil } func (s *S3Store) ReadFile(s3path string, startLine ...int) ([]string, error) { var numLines int var start int - bucketName, prefix, err := s.parsePath(s3path) - if err != nil { - return nil, err - } if len(startLine) == 0 { start = 0 numLines = -1 @@ -51,8 +48,8 @@ func (s *S3Store) ReadFile(s3path string, startLine ...int) ([]string, error) { } resp, err := s.S3Client.GetObject(&s3.GetObjectInput{ - Bucket: aws.String(bucketName), - Key: aws.String(prefix), + Bucket: aws.String(s.Bucket), + Key: aws.String(s3path), }) if err != nil { return nil, err @@ -83,14 +80,9 @@ func (s *S3Store) ReadFile(s3path string, startLine ...int) ([]string, error) { } func (s *S3Store) ListFiles(s3path string) ([]string, error) { - bucketName, prefix, err := s.parsePath(s3path) - if err != nil { - return nil, err - } - resp, err := s.S3Client.ListObjectsV2(&s3.ListObjectsV2Input{ - Bucket: aws.String(bucketName), - Prefix: aws.String(prefix), + Bucket: aws.String(s.Bucket), + Prefix: aws.String(s3path), Delimiter: aws.String("/"), }) if err != nil { @@ -120,16 +112,3 @@ func (s *S3Store) ListBuckets() ([]string, error) { return buckets, nil } - -func (s3 *S3Store) parsePath(s3path string) (bucketName string, prefix string, err error) { - if len(s3path) == 0 { - return "", "", errors.New("the s3path is invalid") - } - parts := strings.SplitN(s3path, "/", 2) - - if len(parts) == 1 { - return parts[0], "", nil - } - - return parts[0], parts[1], nil -} diff --git a/server/api/studio/pkg/filestore/sftpstore.go b/server/api/studio/pkg/filestore/sftpstore.go index da0a494e..53d72c86 100644 --- a/server/api/studio/pkg/filestore/sftpstore.go +++ b/server/api/studio/pkg/filestore/sftpstore.go @@ -4,6 +4,8 @@ import ( "bufio" "errors" "fmt" + "os/user" + "github.com/pkg/sftp" "golang.org/x/crypto/ssh" ) @@ -90,14 +92,20 @@ func (s *SftpStore) ReadFile(path string, startLine ...int) ([]string, error) { func (s *SftpStore) ListFiles(dir string) ([]string, error) { var files []string - walker := s.SftpClient.Walk(dir) - for walker.Step() { - if err := walker.Err(); err != nil { + if dir == "" { + user, err := user.Lookup(s.Username) + if err != nil { return nil, err } - if !walker.Stat().IsDir() { - files = append(files, walker.Path()) - } + dir = user.HomeDir + } + + _files, err := s.SftpClient.ReadDir(dir) + if err != nil { + return nil, err + } + for _, file := range _files { + files = append(files, file.Name()) } return files, nil } diff --git a/server/api/studio/restapi/datasource.api b/server/api/studio/restapi/datasource.api index 150c88ee..85b3bb36 100644 --- a/server/api/studio/restapi/datasource.api +++ b/server/api/studio/restapi/datasource.api @@ -4,6 +4,7 @@ type ( DatasourceS3Config { Endpoint string `json:"endpoint"` Region string `json:"region"` + Bucket string `json:"bucket"` AccessKey string `json:"accessKey"` AccessSecret string `json:"accessSecret"` } @@ -16,27 +17,34 @@ type ( } DatasourceAddRequest { - Type string `json:"type"` - Name string `json:"name"` - S3Config DatasourceS3Config `json:"s3Config,optional"` - SFTPConfig DatasourceSFTPConfig `json:"sftpConfig,optional"` + Type string `json:"type"` + Name string `json:"name"` + S3Config *DatasourceS3Config `json:"s3Config,optional"` + SFTPConfig *DatasourceSFTPConfig `json:"sftpConfig,optional"` } DatasourceAddData { ID int `json:"id"` } + DatasourceListRequest { + Type string `form:"type,optional"` + } + DatasourceRemoveRequest { ID int `path:"id"` } + DatasourceBatchRemoveRequest { + IDs []int `json:"ids"` + } DatasourceConfig { - ID int `json:"id"` - Type string `json:"type"` - Name string `json:"name"` - S3Config DatasourceS3Config `json:"s3Config,optional"` - SFTPConfig DatasourceSFTPConfig `json:"sftpConfig,optional"` - CreateTime int64 `json:"createTime,optional"` + ID int `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + S3Config *DatasourceS3Config `json:"s3Config,optional"` + SFTPConfig *DatasourceSFTPConfig `json:"sftpConfig,optional"` + CreateTime int64 `json:"createTime,optional"` } DatasourceListContentsRequest { @@ -75,9 +83,13 @@ service studio-api { @handler DatasourceRemove delete /api/datasources/:id(DatasourceRemoveRequest) + @doc "Batch Remove Datasource" + @handler DatasourceBatchRemove + delete /api/datasources(DatasourceBatchRemoveRequest) + @doc "List Datasource" @handler DatasourceList - get /api/datasources returns(DatasourceData) + get /api/datasources(DatasourceListRequest) returns(DatasourceData) @doc "List Contents" @handler DatasourceListContents From 5276a1a25480a33bda1f2830a0d52b825bf0cf57 Mon Sep 17 00:00:00 2001 From: hetao92 <18328704+hetao92@users.noreply.github.com> Date: Wed, 15 Mar 2023 09:16:28 +0000 Subject: [PATCH 03/11] mod: update page --- app/config/locale/en-US.json | 6 +- app/config/locale/zh-CN.json | 6 +- .../DatasourceConfig/FileUploadBtn/index.tsx | 58 ++++++++++++ .../LocalFileConfig}/index.module.less | 0 .../LocalFileConfig}/index.tsx | 5 +- .../PlatformConfig}/S3ConfigForm.tsx | 0 .../PlatformConfig}/SftpConfigForm.tsx | 0 .../PlatformConfig}/index.module.less | 0 .../PlatformConfig}/index.tsx | 26 ++++-- .../DatasourceList/LocalFileList/index.tsx | 42 ++------- .../DatasourceList/RemoteList/index.tsx | 2 +- app/pages/Import/TaskList/index.module.less | 74 ++++++++++++++++ app/pages/Import/TaskList/index.tsx | 88 ++++++++++++++----- 13 files changed, 236 insertions(+), 71 deletions(-) create mode 100644 app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx rename app/pages/Import/DatasourceList/{LocalFileList/UploadConfigModal => DatasourceConfig/LocalFileConfig}/index.module.less (100%) rename app/pages/Import/DatasourceList/{LocalFileList/UploadConfigModal => DatasourceConfig/LocalFileConfig}/index.tsx (98%) rename app/pages/Import/DatasourceList/{RemoteList/DatasourceConfigModal => DatasourceConfig/PlatformConfig}/S3ConfigForm.tsx (100%) rename app/pages/Import/DatasourceList/{RemoteList/DatasourceConfigModal => DatasourceConfig/PlatformConfig}/SftpConfigForm.tsx (100%) rename app/pages/Import/DatasourceList/{RemoteList/DatasourceConfigModal => DatasourceConfig/PlatformConfig}/index.module.less (100%) rename app/pages/Import/DatasourceList/{RemoteList/DatasourceConfigModal => DatasourceConfig/PlatformConfig}/index.tsx (76%) diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json index 818e490b..ffa849fe 100644 --- a/app/config/locale/en-US.json +++ b/app/config/locale/en-US.json @@ -278,7 +278,11 @@ "enterAddress": "Enter endpoint", "enterRegion": "Enter region", "serverAddress": "Server Address", - "port": "Port" + "port": "Port", + "newDataSourceTip": "Please add the data source for the first time", + "addNewImport": "Add New Import", + "addNewImportTip": "After adding the data source, create an import task to import the data into the database", + "start": "Start" }, "schema": { "spaceList": "Graph Space List", diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json index ffc4039b..f5a11927 100644 --- a/app/config/locale/zh-CN.json +++ b/app/config/locale/zh-CN.json @@ -279,7 +279,11 @@ "enterAddress": "请输入终端节点地址", "enterRegion": "请输入区域", "serverAddress": "服务器地址", - "port": "端口" + "port": "端口", + "newDataSourceTip": "请先添加数据源", + "addNewImport": "添加导入任务", + "addNewImportTip": "添加数据源后，创建导入任务将数据导入数据库", + "start": "开始" }, "schema": { "spaceList": "图空间列表", diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx new file mode 100644 index 00000000..05159a99 --- /dev/null +++ b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx @@ -0,0 +1,58 @@ +import { useI18n } from '@vesoft-inc/i18n'; +import { message, Upload } from 'antd'; +import React, { PropsWithChildren, useState } from 'react'; +import { StudioFile } from '@app/interfaces/import'; +import { useStore } from '@app/stores'; +import { observer } from 'mobx-react-lite'; +import { debounce } from 'lodash'; +import { getFileSize } from '@app/utils/file'; +import UploadConfigModal from '../LocalFileConfig'; +type IUploadBtnProps = PropsWithChildren<{ + onUpload?: () => void +}> + + +const UploadBtn = (props: IUploadBtnProps) => { + const { files, global } = useStore(); + const { children, onUpload } = props; + const { intl } = useI18n(); + const [previewList, setPreviewList] = useState([]); + const { fileList } = files; + const [visible, setVisible] = useState(false); + const transformFile = async (_file: StudioFile, fileList: StudioFile[]) => { + const size = fileList.reduce((acc, cur) => acc + cur.size, 0); + if(global.gConfig?.maxBytes && size > global.gConfig.maxBytes) { + message.error(intl.get('import.fileSizeLimit', { size: getFileSize(global.gConfig.maxBytes) })); + return false; + } + fileList.forEach(file => { + file.path = `${file.name}`; + file.withHeader = false; + file.delimiter = ','; + }); + setPreviewList(fileList); + setVisible(true); + return false; + }; + const handleConfirm = () => { + onUpload?.(); + setVisible(false); + }; + return ( + <> + {}} + beforeUpload={debounce(transformFile)} + > + {children} + + setVisible(false)} /> + + ); +}; + +export default observer(UploadBtn); diff --git a/app/pages/Import/DatasourceList/LocalFileList/UploadConfigModal/index.module.less b/app/pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig/index.module.less similarity index 100% rename from app/pages/Import/DatasourceList/LocalFileList/UploadConfigModal/index.module.less rename to app/pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig/index.module.less diff --git a/app/pages/Import/DatasourceList/LocalFileList/UploadConfigModal/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig/index.tsx similarity index 98% rename from app/pages/Import/DatasourceList/LocalFileList/UploadConfigModal/index.tsx rename to app/pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig/index.tsx index 0769339d..b54acbd9 100644 --- a/app/pages/Import/DatasourceList/LocalFileList/UploadConfigModal/index.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig/index.tsx @@ -33,7 +33,7 @@ const DelimiterConfigModal = (props: { onConfirm: (string) => void }) => { const UploadConfigModal = (props: IProps) => { const { visible, onConfirm, onCancel, uploadList } = props; const { files } = useStore(); - const { fileList, uploadFile } = files; + const { fileList, uploadFile, getFiles } = files; const { intl } = useI18n(); const state = useLocalObservable(() => ({ data: [], @@ -48,7 +48,7 @@ const UploadConfigModal = (props: IProps) => { const { readRemoteFile } = usePapaParse(); useEffect(() => { const { setState } = state; - visible && setState({ data: uploadList, activeItem: uploadList[0] }); + visible && (getFiles(), setState({ data: uploadList, activeItem: uploadList[0] })); }, [visible]); useEffect(() => { state.activeItem && readFile(); @@ -290,4 +290,5 @@ const UploadConfigModal = (props: IProps) => { ); }; + export default observer(UploadConfigModal); diff --git a/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/S3ConfigForm.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx similarity index 100% rename from app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/S3ConfigForm.tsx rename to app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx diff --git a/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/SftpConfigForm.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/SftpConfigForm.tsx similarity index 100% rename from app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/SftpConfigForm.tsx rename to app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/SftpConfigForm.tsx diff --git a/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.module.less b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less similarity index 100% rename from app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.module.less rename to app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less diff --git a/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx similarity index 76% rename from app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.tsx rename to app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx index 1b9e8fe1..25e642a0 100644 --- a/app/pages/Import/DatasourceList/RemoteList/DatasourceConfigModal/index.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx @@ -4,6 +4,7 @@ import React, { useState } from 'react'; import { IRemoteType } from '@app/interfaces/datasource'; import { useStore } from '@app/stores'; import { observer } from 'mobx-react-lite'; +import UploadLocalBtn from '../FileUploadBtn'; import S3ConfigForm from './S3ConfigForm'; import SftpConfigForm from './SftpConfigForm'; import styles from './index.module.less'; @@ -75,11 +76,26 @@ const DatasourceConfigModal = (props: IProps) => { } }} -
- -
+ + {({ getFieldValue }) => { + const configType = getFieldValue('type'); + if (configType === 'local') { + return
+ + + +
; + } else { + return
+ +
; + } + }} + ); diff --git a/app/pages/Import/DatasourceList/LocalFileList/index.tsx b/app/pages/Import/DatasourceList/LocalFileList/index.tsx index e6ecd853..cc5769a3 100644 --- a/app/pages/Import/DatasourceList/LocalFileList/index.tsx +++ b/app/pages/Import/DatasourceList/LocalFileList/index.tsx @@ -2,25 +2,21 @@ import React, { useCallback, useEffect, useState } from 'react'; import { observer } from 'mobx-react-lite'; import { useStore } from '@app/stores'; import { trackPageView } from '@app/utils/stat'; -import { debounce } from 'lodash'; -import { Button, Popconfirm, Table, Upload, message } from 'antd'; +import { Button, Popconfirm, Table } from 'antd'; import Icon from '@app/components/Icon'; import { getFileSize } from '@app/utils/file'; import cls from 'classnames'; -import { StudioFile } from '@app/interfaces/import'; import { useI18n } from '@vesoft-inc/i18n'; -import UploadConfigModal from './UploadConfigModal'; +import UploadLocalBtn from '../DatasourceConfig/FileUploadBtn'; import PreviewFileModal from './PreviewFileModal'; import styles from './index.module.less'; const FileList = () => { - const { files, global } = useStore(); + const { files } = useStore(); const { intl } = useI18n(); const { fileList, deleteFile, getFiles } = files; const [loading, setLoading] = useState(false); - const [visible, setVisible] = useState(false); - const [previewList, setPreviewList] = useState([]); const [selectFiles, setSelectFiles] = useState([]); const columns = [ { @@ -70,26 +66,6 @@ const FileList = () => { }, }, ]; - const transformFile = async (_file: StudioFile, fileList: StudioFile[]) => { - const size = fileList.reduce((acc, cur) => acc + cur.size, 0); - if(global.gConfig?.maxBytes && size > global.gConfig.maxBytes) { - message.error(intl.get('import.fileSizeLimit', { size: getFileSize(global.gConfig.maxBytes) })); - return false; - } - fileList.forEach(file => { - file.path = `${file.name}`; - file.withHeader = false; - file.delimiter = ','; - }); - setPreviewList(fileList); - setVisible(true); - return false; - }; - - const handleRefresh = () => { - getFileList(); - setVisible(false); - }; const getFileList = async () => { !loading && setLoading(true); @@ -107,18 +83,11 @@ const FileList = () => { return (
- {}} - beforeUpload={debounce(transformFile)} - > + - + { pagination={false} />
- setVisible(false)} />
); }; diff --git a/app/pages/Import/DatasourceList/RemoteList/index.tsx b/app/pages/Import/DatasourceList/RemoteList/index.tsx index 88eaf939..a61724e3 100644 --- a/app/pages/Import/DatasourceList/RemoteList/index.tsx +++ b/app/pages/Import/DatasourceList/RemoteList/index.tsx @@ -9,7 +9,7 @@ import { IRemoteType } from '@app/interfaces/datasource'; import { useI18n } from '@vesoft-inc/i18n'; import dayjs from 'dayjs'; -import DatasourceConfigModal from './DatasourceConfigModal'; +import DatasourceConfigModal from '../DatasourceConfig/PlatformConfig'; import styles from './index.module.less'; interface IProps { diff --git a/app/pages/Import/TaskList/index.module.less b/app/pages/Import/TaskList/index.module.less index 489a7d9d..a7b7616c 100644 --- a/app/pages/Import/TaskList/index.module.less +++ b/app/pages/Import/TaskList/index.module.less @@ -8,6 +8,11 @@ margin-right: 15px; } } + .header { + display: flex; + justify-content: space-between; + align-items: center; + } .taskHeader { color: @lightBlack; font-weight: bold; @@ -16,4 +21,73 @@ border-bottom: 1px solid @gray; margin-bottom: 20px; } +} + +.emptyTip { + padding-top: 100px; + display: flex; + flex-direction: column; + align-items: center; + .box { + border: 1px dashed #828282; + border-radius: 6px; + padding: 22px 30px; + width: 578px; + margin-bottom: 22px; + } + .step { + width: 30px; + height: 30px; + border-radius: 30px; + color: @darkBlue; + background: #D5DDEB; + font-size: 20px; + font-weight: 700; + display: flex; + align-items: center; + justify-content: center; + margin-bottom: 18px; + } + .content { + display: flex; + justify-content: space-between; + align-items: baseline; + } + .title { + font-weight: 500; + font-size: 20px; + margin-bottom: 7px; + } + .tip { + font-weight: 300; + font-size: 12px; + color: #4F4F4F; + } + .btn { + width: 100px; + height: 30px; + } + .arrow { + position: relative; + width: 0; + height: 0; + border-left: 23px solid transparent; + border-right: 23px solid transparent; + border-top: 20px solid #D9D9D9; + left: 50%; + bottom: -55px; + transform: translateX(-50%); + } + + .arrow::before { + content: ""; + position: absolute; + top: -44px; + left: 50%; + margin-left: -13px; + width: 26px; + height: 25px; + background-color: #D9D9D9; + border-radius: 2px; + } } \ No newline at end of file diff --git a/app/pages/Import/TaskList/index.tsx b/app/pages/Import/TaskList/index.tsx index 887ce428..2eba093c 100644 --- a/app/pages/Import/TaskList/index.tsx +++ b/app/pages/Import/TaskList/index.tsx @@ -1,5 +1,5 @@ import { Button, message, Spin } from 'antd'; -import React, { useCallback, useEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useHistory } from 'react-router-dom'; import { observer } from 'mobx-react-lite'; import Icon from '@app/components/Icon'; @@ -7,9 +7,9 @@ import { useStore } from '@app/stores'; import { trackPageView } from '@app/utils/stat'; import { ITaskStatus } from '@app/interfaces/import'; import { useI18n } from '@vesoft-inc/i18n'; +import DatasourceConfigModal from '../DatasourceList/DatasourceConfig/PlatformConfig'; import LogModal from './TaskItem/LogModal'; import TemplateModal from './TemplateModal'; - import styles from './index.module.less'; import TaskItem from './TaskItem'; @@ -36,6 +36,7 @@ const TaskList = (props: IProps) => { const { username, host } = global; const [modalVisible, setVisible] = useState(false); const [importModalVisible, setImportModalVisible] = useState(false); + const [sourceModalVisible, setSourceModalVisible] = useState(false); const [loading, setLoading] = useState(false); const { showTemplateModal = true, showConfigDownload = true, showLogDownload = true } = props; const [logDimension, setLogDimension] = useState({} as ILogDimension); @@ -104,33 +105,71 @@ const TaskList = (props: IProps) => { setLogDimension({} as ILogDimension); } }, [modalVisible]); + const emptyTips = useMemo(() => ([ + { + title: intl.get('import.newDataSource'), + tip: intl.get('import.newDataSourceTip'), + action: () => setSourceModalVisible(true), + btnLabel: intl.get('import.start') + }, + { + title: intl.get('import.addNewImport'), + tip: intl.get('import.addNewImportTip'), + action: () => history.push('/import/create'), + btnLabel: intl.get('import.start') + }, + ]), []); return (
-
- + {showTemplateModal && } +
+ - {showTemplateModal && }

{intl.get('import.taskList')} ({taskList.length})

- - {taskList.map(item => ( - - ))} - + {taskList.length === 0 + ?
+ {emptyTips.map((item, index) => { + return ( +
+

{index + 1}

+
+
+

{item.title}

+

{item.tip}

+
+ +
+ {index !== emptyTips.length - 1 &&
} +
+ ); + })} +
+ : + {taskList.map(item => ( + + ))} + + } {modalVisible && { host={host} onImport={getTaskList} visible={importModalVisible} />} + {sourceModalVisible && setSourceModalVisible(false)} onConfirm={() => setSourceModalVisible(false)} />}
); }; From 2d8d5bb10f67b97af403329e04cadea47adf62c8 Mon Sep 17 00:00:00 2001 From: hetao92 <18328704+hetao92@users.noreply.github.com> Date: Thu, 16 Mar 2023 02:12:09 +0000 Subject: [PATCH 04/11] feat: add importer config --- app/config/locale/en-US.json | 9 ++++-- app/config/locale/zh-CN.json | 9 ++++-- app/interfaces/import.ts | 2 ++ .../DatasourceConfig/FileUploadBtn/index.tsx | 8 +++--- .../PlatformConfig/index.module.less | 6 ++++ .../DatasourceConfig/PlatformConfig/index.tsx | 9 +++++- app/pages/Import/TaskCreate/index.tsx | 28 +++++++++++++++++-- app/utils/constant.ts | 4 ++- app/utils/import.ts | 4 +++ 9 files changed, 67 insertions(+), 12 deletions(-) diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json index ffa849fe..79d93f99 100644 --- a/app/config/locale/en-US.json +++ b/app/config/locale/en-US.json @@ -218,7 +218,7 @@ "importYaml": "Import the YAML file", "templateMatchError": "The {type} in the configuration does not match the current login account/host", "uploadSuccessfully": "Upload files successfully.", - "fileSizeLimit": "The file is too large and exceeds the upload limit({size}), please modify the MaxBytes in the startup configuration file and restart the service", + "fileSizeLimit": "The file is too large and exceeds the upload limit({size}), please upload the file to the data/upload directory under the installation directory via scp", "noHttp": "The address in the configuration file does not support http protocol, please remove http(s)://", "addressMatch": "The address in the configuration file must contain the Graph address of current login connection. Separate multiple addresses with ", "dataSourceFile": "Data source file", @@ -282,7 +282,12 @@ "newDataSourceTip": "Please add the data source for the first time", "addNewImport": "Add New Import", "addNewImportTip": "After adding the data source, create an import task to import the data into the database", - "start": "Start" + "start": "Start", + "s3Tip": "Only support cloud services that compatible with the Amazon S3 interface", + "readerConcurrency": "Reader concurrency", + "readerConcurrencyTip": "The number of concurrent readers that read data from the data source", + "importerConcurrency": "Importer concurrency", + "importerConcurrencyTip": "The number of concurrent importers that import data into Nebula Graph" }, "schema": { "spaceList": "Graph Space List", diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json index f5a11927..7048a8ac 100644 --- a/app/config/locale/zh-CN.json +++ b/app/config/locale/zh-CN.json @@ -218,7 +218,7 @@ "importYaml": "导入 YAML 文件", "templateMatchError": "配置中的{type}与当前登录账号/地址不匹配", "uploadSuccessfully": "上传文件成功", - "fileSizeLimit": "文件过大，超过上传限制({size})，请修改启动配置文件中的 MaxBytes 并重启服务", + "fileSizeLimit": "文件过大，超过上传限制({size})，请将文件通过 scp 的方式上传到安装目录下的 data/upload 目录", "noHttp": "配置文件中的 address 不支持携带 http 协议，请去除 http(s)://", "addressMatch": "配置文件中的 address 字段必须包含当前登录的 Graph 地址。多个地址用“,”隔开。", "dataSourceFile": "文件源", @@ -283,7 +283,12 @@ "newDataSourceTip": "请先添加数据源", "addNewImport": "添加导入任务", "addNewImportTip": "添加数据源后，创建导入任务将数据导入数据库", - "start": "开始" + "start": "开始", + "s3Tip": "只支持兼容Amazon S3接口的云服务", + "readerConcurrency": "读取并发数", + "readerConcurrencyTip": "读取文件的并发数", + "importerConcurrency": "导入并发数", + "importerConcurrencyTip": "导入数据的并发数" }, "schema": { "spaceList": "图空间列表", diff --git a/app/interfaces/import.ts b/app/interfaces/import.ts index 37926f15..f16e98e6 100644 --- a/app/interfaces/import.ts +++ b/app/interfaces/import.ts @@ -57,6 +57,8 @@ export interface IBasicConfig { batchSize?: string; concurrency?: string; retry?: string; + readerConcurrency?: string; + importerConcurrency?: string; } export interface ILogDimension { diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx index 05159a99..807b6911 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx @@ -11,9 +11,9 @@ type IUploadBtnProps = PropsWithChildren<{ onUpload?: () => void }> - +const SizeLimit = 200 * 1024 * 1024; const UploadBtn = (props: IUploadBtnProps) => { - const { files, global } = useStore(); + const { files } = useStore(); const { children, onUpload } = props; const { intl } = useI18n(); const [previewList, setPreviewList] = useState([]); @@ -21,8 +21,8 @@ const UploadBtn = (props: IUploadBtnProps) => { const [visible, setVisible] = useState(false); const transformFile = async (_file: StudioFile, fileList: StudioFile[]) => { const size = fileList.reduce((acc, cur) => acc + cur.size, 0); - if(global.gConfig?.maxBytes && size > global.gConfig.maxBytes) { - message.error(intl.get('import.fileSizeLimit', { size: getFileSize(global.gConfig.maxBytes) })); + if(size > SizeLimit) { + message.error(intl.get('import.fileSizeLimit', { size: getFileSize(SizeLimit) })); return false; } fileList.forEach(file => { diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less index 3f82cacd..68b520a8 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less @@ -18,5 +18,11 @@ width: 180px; } } + .tip { + font-weight: 400; + font-size: 12px; + margin-bottom: 20px; + text-align: center; + } } diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx index 25e642a0..1c7895ca 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx @@ -54,8 +54,15 @@ const DatasourceConfigModal = (props: IProps) => { footer={false} >
+ + {({ getFieldValue }) => { + const configType = type || getFieldValue('type'); + if (configType !== IRemoteType.S3) return; + return

{intl.get('import.s3Tip')}

; + }} + {!type && - {intl.get('import.s3')} - {intl.get('import.sftp')} - {intl.get('import.localFiles')} + {intl.get('import.s3')} + {intl.get('import.sftp')} + {intl.get('import.localFiles')} } {({ getFieldValue }) => { const configType = type || getFieldValue('type'); switch (configType) { - case IRemoteType.S3: + case IDatasourceType.s3: return ; - case IRemoteType.Sftp: + case IDatasourceType.sftp: return ; default: return null; diff --git a/app/pages/Import/DatasourceList/RemoteList/index.tsx b/app/pages/Import/DatasourceList/RemoteList/index.tsx index a61724e3..8e10ec41 100644 --- a/app/pages/Import/DatasourceList/RemoteList/index.tsx +++ b/app/pages/Import/DatasourceList/RemoteList/index.tsx @@ -5,7 +5,7 @@ import { trackPageView } from '@app/utils/stat'; import { Button, message, Popconfirm, Table } from 'antd'; import Icon from '@app/components/Icon'; import cls from 'classnames'; -import { IRemoteType } from '@app/interfaces/datasource'; +import { IDatasourceType } from '@app/interfaces/datasource'; import { useI18n } from '@vesoft-inc/i18n'; import dayjs from 'dayjs'; @@ -13,7 +13,7 @@ import DatasourceConfigModal from '../DatasourceConfig/PlatformConfig'; import styles from './index.module.less'; interface IProps { - type: IRemoteType; + type: IDatasourceType; } const cloudKeys = [ { @@ -54,8 +54,8 @@ const sftpKeys = [{ } ]; const columnKeys = { - [IRemoteType.S3]: cloudKeys, - [IRemoteType.Sftp]: sftpKeys, + [IDatasourceType.s3]: cloudKeys, + [IDatasourceType.sftp]: sftpKeys, }; const DatasourceList = (props: IProps) => { diff --git a/app/pages/Import/DatasourceList/index.tsx b/app/pages/Import/DatasourceList/index.tsx index 66ce68a6..2383c2a5 100644 --- a/app/pages/Import/DatasourceList/index.tsx +++ b/app/pages/Import/DatasourceList/index.tsx @@ -2,7 +2,7 @@ import React from 'react'; import { observer } from 'mobx-react-lite'; import { Tabs, TabsProps } from 'antd'; import { useI18n } from '@vesoft-inc/i18n'; -import { IRemoteType } from '@app/interfaces/datasource'; +import { IDatasourceType } from '@app/interfaces/datasource'; import LocalFileList from './LocalFileList'; import RemoteList from './RemoteList'; import styles from './index.module.less'; @@ -11,24 +11,24 @@ const DatasourceList = () => { const { intl } = useI18n(); const items: TabsProps['items'] = [ { - key: 'local', + key: IDatasourceType.local, label: intl.get('import.localFiles'), children: }, { - key: 's3', + key: IDatasourceType.s3, label: intl.get('import.s3'), - children: + children: }, { - key: 'sftp', + key: IDatasourceType.sftp, label: intl.get('import.sftp'), - children: + children: }, ]; return (
- +
); }; diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelectModal.tsx b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelectModal.tsx new file mode 100644 index 00000000..148646cc --- /dev/null +++ b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelectModal.tsx @@ -0,0 +1,174 @@ +import { ArrowLeftOutlined, FileTextFilled, FolderFilled, SyncOutlined } from '@ant-design/icons'; +import { IDatasourceType } from '@app/interfaces/datasource'; +import { useStore } from '@app/stores'; +import { useBatchState } from '@app/utils'; +import { getFileSize } from '@app/utils/file'; +import { useI18n } from '@vesoft-inc/i18n'; +import { Button, Modal, Select, Spin } from 'antd'; +import cls from 'classnames'; +import { observer } from 'mobx-react-lite'; +import React, { useEffect } from 'react'; + +import styles from './index.module.less'; +const Option = Select.Option; +interface IProps { + visible: boolean; + onConfirm: (password: string) => void + onCancel: () => void; +} +const ConfigConfirmModal = (props: IProps) => { + const { visible, onConfirm, onCancel } = props; + const { datasource, files } = useStore(); + const { getFiles } = files; + const { getDatasourceList, getDatasourceDetail, previewFile } = datasource; + const { intl } = useI18n(); + const { state, setState } = useBatchState({ + loading: false, + options: [], + directory: [], + path: '', + activeId: null, + activeItem: null, + }); + const { options, directory, path, activeItem, activeId, loading } = state; + const handleConfirm = (password?: string) => { + onConfirm(password); + }; + const handleCancel = () => { + onCancel(); + }; + + const getLocalFiles = async () => { + const files = await getFiles(); + setState({ + directory: files, + path: '/', + loading: false + }); + }; + const getDatasourceDirectory = async (id, path?) => { + const data = await getDatasourceDetail({ id, path }); + setState({ + directory: data, + activeId: id, + path: path || '/', + loading: false + }); + }; + const handleTypeChange = async (value) => { + setState({ loading: true }); + if(value === IDatasourceType.local) { + getLocalFiles(); + } else { + getDatasourceDirectory(value); + } + }; + const init = async () => { + setState({ loading: true }); + const data = await getDatasourceList(); + setState({ + options: data, + loading: false + }); + }; + const handleSelectFile = async (item) => { + setState({ loading: true }); + const newPath = `${path === '/' ? '' : path}${item.name}${item.type === 'directory' ? '/' : ''}`; + if(item.type === 'directory') { + getDatasourceDirectory(activeId, newPath); + } else { + await previewFile({ id: activeId, path: newPath }); + setState({ + loading: false + }); + } + }; + const handlePathBack = async () => { + if(!path || path === '/') return; + setState({ loading: true }); + const _path = path.slice(0, -1).split('/'); + _path.pop(); + const newPath = _path.join('/').length ? _path.join('/') + '/' : ''; + getDatasourceDirectory(activeId, newPath); + }; + + const handleRefresh = async () => { + setState({ loading: true }); + if (!activeId) { + getLocalFiles(); + return; + } + const _path = !path || path === '/' ? '' : path; + getDatasourceDirectory(activeId, _path); + }; + useEffect(() => { + init(); + }, []); + return ( + handleCancel()} + className={styles.selectFileModal} + footer={false} + width={700} + > + +
+ {intl.get('import.datasourceType')} + +
+
+ {intl.get('import.filePath')} +
+
{path}
+
+
+
+
+
+ {directory.map((item) => ( +
setState({ activeItem: item })} + onDoubleClick={() => handleSelectFile(item)} + > + {item.type === 'directory' ? : } +
+ {item.name} + {item.type === 'directory' ? intl.get('import.directory') : getFileSize(item.size)} +
+
+ ))} +
+
+ +
+ + + + ); +}; + +export default observer(ConfigConfirmModal); diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.module.less b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.module.less new file mode 100644 index 00000000..e936a3e2 --- /dev/null +++ b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.module.less @@ -0,0 +1,133 @@ +.selectFileModal { + .label { + font-weight: 700; + font-size: 12px; + margin-right: 5px; + min-width: 110px; + user-select:none; + &::after { + content: ':'; + margin-left: 2px; + } + } + .row { + margin-bottom: 16px; + display: flex; + align-items: center; + } + .typeSelect { + min-width: 300px; + } + .operations { + display: flex; + border: 1px solid #D5DDEB; + border-radius: 3px; + .path { + min-width: 470px; + padding-left: 12px; + display: inline-flex; + align-items: center; + user-select: none; + } + .btn { + cursor: pointer; + width: 38px; + height: 38px; + display: flex; + align-items: center; + justify-content: center; + border-left: 1px solid #D5DDEB; + svg { + width: 16px; + height: 16px; + } + &.disabled { + cursor: not-allowed; + } + } + } + .fileDirectory { + background: #FFFFFF; + border: 1px solid #E0E0E0; + min-height: 280px; + padding: 12px; + display: grid; + grid-template-columns: repeat(3, minmax(calc((100% - 24px) / 3), 1fr)); + grid-gap: 10px; + grid-template-rows: max-content; + margin-bottom: 12px; + align-content: baseline; + } + .item { + background: #FFFFFF; + border: 1px solid #E0E0E0; + height: 56px; + display: flex; + align-items: center; + padding: 9px 11px; + cursor: pointer; + .icon svg { + width: 40px; + height: 40px; + } + .content { + display: flex; + flex-direction: column; + margin-left: 5px; + user-select:none; + } + .title { + font-weight: 500; + font-size: 14px; + overflow: hidden; + text-overflow: ellipsis; + max-width: 130px; + white-space: nowrap; + } + .desc { + font-weight: 400; + font-size: 12px; + line-height: 14px; + color: #BDBDBD; + } + &.actived { + border: 1px solid #0D8BFF; + box-shadow: 0px 1px 4px rgba(0, 0, 0, 0.25); + } + } +} + +.btns { + text-align: center; + :global(.ant-btn) { + width: 180px; + } +} +.typeOptions { + :global(.ant-select-item-option-content) { + font-weight: 500; + font-size: 14px; + color: #2F3A4A; + } +} +.typeItem { + display: flex; + align-items: center; + justify-content: space-between; + + .value { + font-weight: 500; + font-size: 14px; + color: #2F3A4A; + max-width: 150px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + .type { + font-weight: 500; + font-size: 12px; + line-height: 14px; + color: #8697B0; + } +} \ No newline at end of file diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx index c5bae406..69cb34ad 100644 --- a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx +++ b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx @@ -1,5 +1,5 @@ -import { Collapse, Select, Table, Tooltip } from 'antd'; -import React, { useMemo } from 'react'; +import { Button, Collapse, Select, Table, Tooltip } from 'antd'; +import React, { useMemo, useState } from 'react'; import { observer } from 'mobx-react-lite'; import cls from 'classnames'; import { useStore } from '@app/stores'; @@ -11,6 +11,7 @@ import { ISchemaEnum } from '@app/interfaces/schema'; import { IEdgeFileItem, ITagFileItem } from '@app/stores/import'; import { IImportFile } from '@app/interfaces/import'; import styles from '../index.module.less'; +import FileSelectModal from './FileSelectModal'; const Option = Select.Option; const Panel = Collapse.Panel; @@ -99,6 +100,7 @@ const FileMapping = (props: IProps) => { const { fileList, getFiles } = files; const { file, props: mappingProps } = item; const { intl } = useI18n(); + const [visible, setVisible] = useState(false); const handleFileChange = (value: string) => { const file = fileList.find(item => item.name === value); onReset(item, file); @@ -145,13 +147,15 @@ const FileMapping = (props: IProps) => { } }; + const idConfig = useMemo(() => type === ISchemaEnum.Tag ? idMap[ISchemaEnum.Tag] : idMap[ISchemaEnum.Edge], [type]); return (
{intl.get('import.dataSourceFile')} - { {file.name} ))} - + */}
onRemove(item)} />
@@ -176,6 +180,7 @@ const FileMapping = (props: IProps) => { rowKey="name" pagination={false} /> + {visible && setVisible(false)} onConfirm={() => {}} />}
); }; diff --git a/app/pages/Import/TaskList/index.tsx b/app/pages/Import/TaskList/index.tsx index 2eba093c..7b615073 100644 --- a/app/pages/Import/TaskList/index.tsx +++ b/app/pages/Import/TaskList/index.tsx @@ -139,7 +139,7 @@ const TaskList = (props: IProps) => {

{intl.get('import.taskList')} ({taskList.length})

- {taskList.length === 0 + {!loading && taskList.length === 0 ?
{emptyTips.map((item, index) => { return ( diff --git a/app/stores/datasource.ts b/app/stores/datasource.ts index 90d61cbc..caa3a05a 100644 --- a/app/stores/datasource.ts +++ b/app/stores/datasource.ts @@ -37,24 +37,22 @@ export class DatasourceStore { return code === 0; }; getDatasourceDetail = async (payload: { - id: string, + id: number, path?: string, }) => { - const { id } = payload; - const { code, data } = await service.getDatasourceDetail({ id }); + const { id, path } = payload; + const { code, data } = await service.getDatasourceDetail({ id, path }); if(code === 0) { - console.log('data', data); - return data; + return data.list; } }; previewFile = async (payload: { id: string, path?: string, }) => { - const { id } = payload; - const { code, data } = await service.previewFile({ id, path: 'importer-hetao-test/player.csv' }); + const { id, path } = payload; + const { code, data } = await service.previewFile({ id, path }); if(code === 0) { - console.log('data', data); return data; } }; diff --git a/app/stores/files.ts b/app/stores/files.ts index 3a349734..4296b79f 100644 --- a/app/stores/files.ts +++ b/app/stores/files.ts @@ -34,6 +34,7 @@ export class FilesStore { this.update({ fileList: data.list || [], }); + return data.list; } }; uploadFile = async (files: StudioFile[]) => { diff --git a/server/api/studio/internal/service/datasource.go b/server/api/studio/internal/service/datasource.go index b8b83968..d9696263 100644 --- a/server/api/studio/internal/service/datasource.go +++ b/server/api/studio/internal/service/datasource.go @@ -187,11 +187,19 @@ func (d *datasourceService) ListContents(request types.DatasourceListContentsReq return nil, err } fileList, err := store.ListFiles(request.Path) + list := make([]types.FileConfig, 0) + for _, item := range fileList { + list = append(list, types.FileConfig{ + Name: item.Name, + Size: item.Size, + Type: item.Type, + }) + } if err != nil { return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err, "listFiles failed") } return &types.DatasourceListContentsData{ - List: fileList, + List: list, }, nil } diff --git a/server/api/studio/internal/types/types.go b/server/api/studio/internal/types/types.go index 9683dc31..197ca8d2 100644 --- a/server/api/studio/internal/types/types.go +++ b/server/api/studio/internal/types/types.go @@ -393,8 +393,14 @@ type DatasourceListContentsRequest struct { Path string `form:"path,optional"` } +type FileConfig struct { + Size int64 `json:"size"` + Type string `json:"type"` + Name string `json:"name"` +} + type DatasourceListContentsData struct { - List []string `json:"list"` + List []FileConfig `json:"list"` } type DatasourceData struct { diff --git a/server/api/studio/pkg/filestore/filestore.go b/server/api/studio/pkg/filestore/filestore.go index b1ed5975..5d856aa0 100644 --- a/server/api/studio/pkg/filestore/filestore.go +++ b/server/api/studio/pkg/filestore/filestore.go @@ -8,7 +8,13 @@ import ( type ( FileStore interface { ReadFile(path string, startLine ...int) ([]string, error) - ListFiles(dir string) ([]string, error) + ListFiles(dir string) ([]FileConfig, error) + } + + FileConfig struct { + Type string + Name string + Size int64 } SftpConfig struct { diff --git a/server/api/studio/pkg/filestore/s3store.go b/server/api/studio/pkg/filestore/s3store.go index 073ce6d0..6145abfa 100644 --- a/server/api/studio/pkg/filestore/s3store.go +++ b/server/api/studio/pkg/filestore/s3store.go @@ -3,6 +3,7 @@ package filestore import ( "bufio" "errors" + "strings" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -79,7 +80,7 @@ func (s *S3Store) ReadFile(s3path string, startLine ...int) ([]string, error) { return lines, nil } -func (s *S3Store) ListFiles(s3path string) ([]string, error) { +func (s *S3Store) ListFiles(s3path string) ([]FileConfig, error) { resp, err := s.S3Client.ListObjectsV2(&s3.ListObjectsV2Input{ Bucket: aws.String(s.Bucket), Prefix: aws.String(s3path), @@ -88,13 +89,30 @@ func (s *S3Store) ListFiles(s3path string) ([]string, error) { if err != nil { return nil, err } - - var files []string + var files []FileConfig for _, obj := range resp.CommonPrefixes { - files = append(files, *obj.Prefix) + name := (*obj.Prefix)[:len(*obj.Prefix)-1] // remove trailing slash + files = append(files, FileConfig{ + Name: strings.TrimPrefix(name, s3path), + Type: "directory", + }) } for _, obj := range resp.Contents { - files = append(files, *obj.Key) + var objType string + key := *obj.Key + if key[len(*obj.Key)-1:] == "/" { + objType = "directory" + } else if strings.HasSuffix(key, ".csv") { + objType = "csv" + } + if objType != "" { + s3Object := FileConfig{ + Name: strings.TrimPrefix(key, s3path), + Type: objType, + Size: *obj.Size, + } + files = append(files, s3Object) + } } return files, nil diff --git a/server/api/studio/pkg/filestore/sftpstore.go b/server/api/studio/pkg/filestore/sftpstore.go index 53d72c86..93e9db47 100644 --- a/server/api/studio/pkg/filestore/sftpstore.go +++ b/server/api/studio/pkg/filestore/sftpstore.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os/user" + "strings" "github.com/pkg/sftp" "golang.org/x/crypto/ssh" @@ -90,8 +91,8 @@ func (s *SftpStore) ReadFile(path string, startLine ...int) ([]string, error) { return lines, nil } -func (s *SftpStore) ListFiles(dir string) ([]string, error) { - var files []string +func (s *SftpStore) ListFiles(dir string) ([]FileConfig, error) { + var files []FileConfig if dir == "" { user, err := user.Lookup(s.Username) if err != nil { @@ -105,7 +106,22 @@ func (s *SftpStore) ListFiles(dir string) ([]string, error) { return nil, err } for _, file := range _files { - files = append(files, file.Name()) + isDir := file.IsDir() + name := file.Name() + var fileType string + if isDir { + fileType = "directory" + } else if strings.HasSuffix(name, ".csv") { + fileType = "csv" + } + if fileType != "" { + files = append(files, FileConfig{ + Name: name, + Size: file.Size(), + Type: fileType, + }) + } + } return files, nil } diff --git a/server/api/studio/restapi/datasource.api b/server/api/studio/restapi/datasource.api index 85b3bb36..5e7442ab 100644 --- a/server/api/studio/restapi/datasource.api +++ b/server/api/studio/restapi/datasource.api @@ -52,8 +52,14 @@ type ( Path string `form:"path,optional"` } + FileConfig { + Size int64 `json:"size"` + Type string `json:"type"` + Name string `json:"name"` + } + DatasourceListContentsData { - List []string `json:"list"` + List []FileConfig `json:"list"` } DatasourceData { From 26a68e6a9201d0c1f301175a983a3cd9063b01e3 Mon Sep 17 00:00:00 2001 From: hetao92 <18328704+hetao92@users.noreply.github.com> Date: Fri, 17 Mar 2023 10:09:50 +0000 Subject: [PATCH 06/11] mod: update file select logic before import --- .../FileConfigSetting}/index.module.less | 5 - .../FileConfigSetting}/index.tsx | 102 +++---- app/config/locale/en-US.json | 3 +- app/config/locale/zh-CN.json | 3 +- .../FileUploadBtn/index.module.less | 5 + .../DatasourceConfig/FileUploadBtn/index.tsx | 28 +- .../FileMapping/FileSelectModal.tsx | 269 ++++++++++-------- .../SchemaConfig/FileMapping/index.tsx | 46 ++- .../TaskCreate/SchemaConfig/index.module.less | 24 +- 9 files changed, 284 insertions(+), 201 deletions(-) rename app/{pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig => components/FileConfigSetting}/index.module.less (96%) rename app/{pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig => components/FileConfigSetting}/index.tsx (82%) create mode 100644 app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.module.less diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig/index.module.less b/app/components/FileConfigSetting/index.module.less similarity index 96% rename from app/pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig/index.module.less rename to app/components/FileConfigSetting/index.module.less index 0f7ec898..470fabf0 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig/index.module.less +++ b/app/components/FileConfigSetting/index.module.less @@ -1,10 +1,5 @@ @import '~@app/common.less'; -.uploadModal { - :global(.ant-modal-body) { - padding: 0 0 20px; - } -} .container { display: flex; border-bottom: 1px solid #D5DDEB; diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig/index.tsx b/app/components/FileConfigSetting/index.tsx similarity index 82% rename from app/pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig/index.tsx rename to app/components/FileConfigSetting/index.tsx index b54acbd9..eeed1177 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/LocalFileConfig/index.tsx +++ b/app/components/FileConfigSetting/index.tsx @@ -1,22 +1,21 @@ import Icon from '@app/components/Icon'; import { useI18n } from '@vesoft-inc/i18n'; -import { Button, Input, Modal, Table, Popconfirm, Dropdown, message } from 'antd'; +import { Button, Input, Modal, Table, Popconfirm, Dropdown } from 'antd'; import { v4 as uuidv4 } from 'uuid'; import React, { useCallback, useEffect, useState } from 'react'; import { usePapaParse } from 'react-papaparse'; import cls from 'classnames'; import { StudioFile } from '@app/interfaces/import'; -import { useStore } from '@app/stores'; import { observer, useLocalObservable } from 'mobx-react-lite'; import { ExclamationCircleFilled } from '@ant-design/icons'; import { observable } from 'mobx'; import Checkbox, { CheckboxChangeEvent } from 'antd/lib/checkbox'; import styles from './index.module.less'; interface IProps { - visible: boolean; - onConfirm: () => void; + onConfirm: (data) => void; onCancel: () => void; - uploadList: StudioFile[]; + preUploadList: StudioFile[]; + duplicateCheckList?: StudioFile[]; } const DelimiterConfigModal = (props: { onConfirm: (string) => void }) => { @@ -30,10 +29,8 @@ const DelimiterConfigModal = (props: { onConfirm: (string) => void }) => {
); }; -const UploadConfigModal = (props: IProps) => { - const { visible, onConfirm, onCancel, uploadList } = props; - const { files } = useStore(); - const { fileList, uploadFile, getFiles } = files; +const FileConfigSetting = (props: IProps) => { + const { onConfirm, onCancel, preUploadList, duplicateCheckList } = props; const { intl } = useI18n(); const state = useLocalObservable(() => ({ data: [], @@ -44,12 +41,17 @@ const UploadConfigModal = (props: IProps) => { loading: false, uploading: false, setState: (obj) => Object.assign(state, obj), - }), { data: observable.ref }); - const { readRemoteFile } = usePapaParse(); + }), { data: observable.ref, activeItem: observable.ref }); + const { readRemoteFile, readString } = usePapaParse(); useEffect(() => { const { setState } = state; - visible && (getFiles(), setState({ data: uploadList, activeItem: uploadList[0] })); - }, [visible]); + setState({ + data: preUploadList, + activeItem: preUploadList[0], + checkAll: preUploadList.every((item) => item.withHeader), + indeterminate: preUploadList.some((item) => item.withHeader) && !preUploadList.every((item) => item.withHeader), + }); + }, []); useEffect(() => { state.activeItem && readFile(); }, [state.activeItem]); @@ -57,21 +59,36 @@ const UploadConfigModal = (props: IProps) => { const { activeItem, setState } = state; if(!activeItem) return; setState({ loading: true }); - const url = URL.createObjectURL(activeItem); let content = []; - readRemoteFile(url, { - delimiter: activeItem.delimiter, - download: true, - preview: 5, - worker: true, - skipEmptyLines: true, - step: (row) => { - content = [...content, row.data]; - }, - complete: () => { - setState({ loading: false, previewContent: content }); - } - }); + if(activeItem.sample) { + readString(activeItem.sample, { + delimiter: activeItem.delimiter || ',', + worker: true, + skipEmptyLines: true, + step: (row) => { + content = [...content, row.data]; + }, + complete: () => { + setState({ loading: false, previewContent: content }); + } + }); + } else { + const url = URL.createObjectURL(activeItem); + readRemoteFile(url, { + delimiter: activeItem.delimiter, + download: true, + preview: 5, + worker: true, + skipEmptyLines: true, + step: (row) => { + content = [...content, row.data]; + }, + complete: () => { + setState({ loading: false, previewContent: content }); + } + }); + } + }, []); const onCheckAllChange = useCallback((e: CheckboxChangeEvent) => { @@ -80,7 +97,7 @@ const UploadConfigModal = (props: IProps) => { setState({ checkAll: checked, indeterminate: false, - data: data.map(i => (i.withHeader = checked, i)) + data: data.map(i => (i.withHeader = checked, i)), }); }, []); @@ -127,7 +144,7 @@ const UploadConfigModal = (props: IProps) => { const handleConfirm = useCallback(() => { const { data } = state; - const existFileName = fileList.map((file) => file.name); + const existFileName = duplicateCheckList?.map((file) => file.name) || []; const repeatFiles = data.filter((file) => existFileName.includes(file.name)); if(!repeatFiles.length) { startImport(); @@ -152,15 +169,11 @@ const UploadConfigModal = (props: IProps) => { startImport(); }, }); - }, [fileList]); + }, [duplicateCheckList]); const startImport = useCallback(async () => { const { data, setState } = state; setState({ uploading: true }); - const res = await uploadFile(data); - if(res.code === 0) { - onConfirm(); - message.success(intl.get('import.uploadSuccessfully')); - } + await onConfirm(data); setState({ uploading: false }); }, []); const handleCancel = useCallback(() => { @@ -168,9 +181,6 @@ const UploadConfigModal = (props: IProps) => { !uploading && onCancel(); }, []); - if(!visible) { - return null; - } const { uploading, data, activeItem, previewContent, loading, setState, checkAll, indeterminate } = state; const parseColumns = previewContent.length ? previewContent[0].map((header, index) => { @@ -229,16 +239,8 @@ const UploadConfigModal = (props: IProps) => { ), }, ]; - return ( - handleCancel()} - className={styles.uploadModal} - footer={false} - > +
{ className={styles.previewTable} dataSource={data} columns={columns} - rowKey="uid" + rowKey={() => uuidv4()} pagination={false} /> @@ -286,9 +288,9 @@ const UploadConfigModal = (props: IProps) => { {intl.get('common.confirm')} - + ); }; -export default observer(UploadConfigModal); +export default observer(FileConfigSetting); diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json index 828cb2ed..90e58350 100644 --- a/app/config/locale/en-US.json +++ b/app/config/locale/en-US.json @@ -291,7 +291,8 @@ "selectDatasourceFile": "Select Data Source file", "datasourceType": "Data Source Type", "filePath": "File Path", - "directory": "Directory" + "directory": "Directory", + "preview": "Preview" }, "schema": { "spaceList": "Graph Space List", diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json index 1b0af932..5295dc59 100644 --- a/app/config/locale/zh-CN.json +++ b/app/config/locale/zh-CN.json @@ -292,7 +292,8 @@ "selectDatasourceFile": "选择数据源文件", "datasourceType": "数据源类型", "filePath": "文件路径", - "directory": "目录" + "directory": "目录", + "preview": "预览" }, "schema": { "spaceList": "图空间列表", diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.module.less b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.module.less new file mode 100644 index 00000000..377ae04d --- /dev/null +++ b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.module.less @@ -0,0 +1,5 @@ +.uploadModal { + :global(.ant-modal-body) { + padding: 0 0 20px; + } +} \ No newline at end of file diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx index 807b6911..f3f89b26 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx @@ -1,12 +1,13 @@ import { useI18n } from '@vesoft-inc/i18n'; -import { message, Upload } from 'antd'; +import { message, Modal, Upload } from 'antd'; import React, { PropsWithChildren, useState } from 'react'; import { StudioFile } from '@app/interfaces/import'; import { useStore } from '@app/stores'; import { observer } from 'mobx-react-lite'; import { debounce } from 'lodash'; import { getFileSize } from '@app/utils/file'; -import UploadConfigModal from '../LocalFileConfig'; +import FileConfigSetting from '@app/components/FileConfigSetting'; +import styles from './index.module.less'; type IUploadBtnProps = PropsWithChildren<{ onUpload?: () => void }> @@ -17,7 +18,7 @@ const UploadBtn = (props: IUploadBtnProps) => { const { children, onUpload } = props; const { intl } = useI18n(); const [previewList, setPreviewList] = useState([]); - const { fileList } = files; + const { fileList, uploadFile } = files; const [visible, setVisible] = useState(false); const transformFile = async (_file: StudioFile, fileList: StudioFile[]) => { const size = fileList.reduce((acc, cur) => acc + cur.size, 0); @@ -34,7 +35,10 @@ const UploadBtn = (props: IUploadBtnProps) => { setVisible(true); return false; }; - const handleConfirm = () => { + const handleConfirm = async (data) => { + const res = await uploadFile(data); + if (res.code !== 0) return; + message.success(intl.get('import.uploadSuccessfully')); onUpload?.(); setVisible(false); }; @@ -50,7 +54,21 @@ const UploadBtn = (props: IUploadBtnProps) => { > {children} - setVisible(false)} /> + setVisible(false)} + className={styles.uploadModal} + footer={false} + > + setVisible(false)} /> + ); }; diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelectModal.tsx b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelectModal.tsx index 148646cc..49b5c2e9 100644 --- a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelectModal.tsx +++ b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelectModal.tsx @@ -7,46 +7,54 @@ import { useI18n } from '@vesoft-inc/i18n'; import { Button, Modal, Select, Spin } from 'antd'; import cls from 'classnames'; import { observer } from 'mobx-react-lite'; -import React, { useEffect } from 'react'; - +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import FileConfigSetting from '@app/components/FileConfigSetting'; import styles from './index.module.less'; const Option = Select.Option; -interface IProps { +interface IModalProps { visible: boolean; - onConfirm: (password: string) => void + onConfirm: (file, cachedDatasourceState) => void onCancel: () => void; + cachedDatasourceState?: any; +} +interface IFileSelect { + onConfirm: (file, cachedState) => void, + cachedState?: any } -const ConfigConfirmModal = (props: IProps) => { - const { visible, onConfirm, onCancel } = props; +const FileSelect = observer((props: IFileSelect) => { + const { intl } = useI18n(); const { datasource, files } = useStore(); - const { getFiles } = files; + const { onConfirm, cachedState } = props; const { getDatasourceList, getDatasourceDetail, previewFile } = datasource; - const { intl } = useI18n(); + const { getFiles } = files; const { state, setState } = useBatchState({ loading: false, options: [], - directory: [], - path: '', - activeId: null, - activeItem: null, + directory: cachedState?.directory || [], + path: cachedState?.path || '', + activeId: cachedState?.activeId, + activeItem: cachedState?.activeItem, }); const { options, directory, path, activeItem, activeId, loading } = state; - const handleConfirm = (password?: string) => { - onConfirm(password); - }; - const handleCancel = () => { - onCancel(); - }; + const init = useCallback(async () => { + setState({ loading: true }); + const data = await getDatasourceList(); + setState({ + options: data, + loading: false, + }); + }, []); - const getLocalFiles = async () => { + const getLocalFiles = useCallback(async () => { const files = await getFiles(); setState({ directory: files, path: '/', - loading: false + loading: false, + activeId: IDatasourceType.local }); - }; - const getDatasourceDirectory = async (id, path?) => { + }, []); + const getDatasourceDirectory = useCallback(async (id, path?) => { const data = await getDatasourceDetail({ id, path }); setState({ directory: data, @@ -54,45 +62,42 @@ const ConfigConfirmModal = (props: IProps) => { path: path || '/', loading: false }); - }; - const handleTypeChange = async (value) => { - setState({ loading: true }); - if(value === IDatasourceType.local) { - getLocalFiles(); - } else { - getDatasourceDirectory(value); - } - }; - const init = async () => { - setState({ loading: true }); - const data = await getDatasourceList(); - setState({ - options: data, - loading: false - }); - }; - const handleSelectFile = async (item) => { + }, []); + + useEffect(() => { + init(); + }, []); + const handleSelectFile = useCallback(async (item) => { setState({ loading: true }); const newPath = `${path === '/' ? '' : path}${item.name}${item.type === 'directory' ? '/' : ''}`; + if (item.type !== 'directory') return; if(item.type === 'directory') { getDatasourceDirectory(activeId, newPath); - } else { - await previewFile({ id: activeId, path: newPath }); - setState({ - loading: false - }); } - }; - const handlePathBack = async () => { + }, [path]); + + const handlePathBack = useCallback(async () => { if(!path || path === '/') return; setState({ loading: true }); const _path = path.slice(0, -1).split('/'); _path.pop(); const newPath = _path.join('/').length ? _path.join('/') + '/' : ''; getDatasourceDirectory(activeId, newPath); - }; - - const handleRefresh = async () => { + }, [path, activeId]); + const handleTypeChange = useCallback(async (value) => { + setState({ + loading: true, + activeId: null, + activeItem: null, + path: '', + }); + if(value === IDatasourceType.local) { + getLocalFiles(); + } else { + getDatasourceDirectory(value); + } + }, []); + const handleRefresh = useCallback(async () => { setState({ loading: true }); if (!activeId) { getLocalFiles(); @@ -100,75 +105,117 @@ const ConfigConfirmModal = (props: IProps) => { } const _path = !path || path === '/' ? '' : path; getDatasourceDirectory(activeId, _path); + }, [activeId, path]); + const handleConfirm = useCallback(async () => { + if(activeItem && activeId === IDatasourceType.local) { + // select local file + onConfirm(activeItem, state); + } else { + setState({ loading: true }); + const _path = `${path === '/' ? '' : path}${activeItem.name}`; + const data = await previewFile({ id: activeId, path: _path }); + const item = { + name: activeItem.name, + withHeader: false, + delimiter: ',', + sample: data.contents.join('\r\n'), + }; + setState({ loading: false }); + onConfirm(item, state); + } + }, [activeItem, activeId, path]); + return +
+ {intl.get('import.datasourceType')} + +
+
+ {intl.get('import.filePath')} +
+
{path}
+
+
+
+
+
+ {directory.map((item) => ( +
setState({ activeItem: item })} + onDoubleClick={() => handleSelectFile(item)} + > + {item.type === 'directory' ? : } +
+ {item.name} + {item.type === 'directory' ? intl.get('import.directory') : getFileSize(item.size)} +
+
+ ))} +
+
+ +
+ ; +}); +const FileSelectModal = (props: IModalProps) => { + const { visible, onConfirm, onCancel, cachedDatasourceState } = props; + const { intl } = useI18n(); + const [step, setStep] = useState(0); + const [preUploadList, setPreUploadList] = useState([]); + const [cachedState, setcachedState] = useState(cachedDatasourceState || undefined); + const title = useMemo(() => { + if(step === 0) { + return intl.get('import.selectDatasourceFile'); + } + return intl.get('import.preview'); + }, [step]); + const handlePreview = (file, cachedState) => { + setPreUploadList([file]); + setcachedState(cachedState); + setStep(1); + }; + const handleConfirm = (file) => { + onConfirm(file[0], cachedState); }; - useEffect(() => { - init(); - }, []); return ( handleCancel()} + onCancel={onCancel} className={styles.selectFileModal} footer={false} - width={700} + width={step === 0 ? 700 : 920} > - -
- {intl.get('import.datasourceType')} - -
-
- {intl.get('import.filePath')} -
-
{path}
-
-
-
-
-
- {directory.map((item) => ( -
setState({ activeItem: item })} - onDoubleClick={() => handleSelectFile(item)} - > - {item.type === 'directory' ? : } -
- {item.name} - {item.type === 'directory' ? intl.get('import.directory') : getFileSize(item.size)} -
-
- ))} -
-
- -
- - + {step === 0 && } + {step === 1 && setStep(0)} />} ); }; -export default observer(ConfigConfirmModal); +export default observer(FileSelectModal); diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx index 69cb34ad..43edaf11 100644 --- a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx +++ b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx @@ -96,15 +96,13 @@ const idMap = { const FileMapping = (props: IProps) => { const { item, onRemove, type, onReset } = props; - const { files } = useStore(); - const { fileList, getFiles } = files; const { file, props: mappingProps } = item; const { intl } = useI18n(); const [visible, setVisible] = useState(false); - const handleFileChange = (value: string) => { - const file = fileList.find(item => item.name === value); - onReset(item, file); - }; + const [selectFile, setSelectFile] = useState({ + file: null, + cachedState: null, + }); const updateFilePropMapping = (index: number, value: number) => item.updatePropItem(index, { mapping: value }); const columns = [ @@ -141,12 +139,12 @@ const FileMapping = (props: IProps) => { dataIndex: 'type', }, ]; - const handleGetFiles = () => { - if(fileList.length === 0) { - getFiles(); - } - }; + const handleUpdateFile = (file, cachedState) => { + setSelectFile({ file, cachedState }); + onReset(item, file); + setVisible(false); + }; const idConfig = useMemo(() => type === ISchemaEnum.Tag ? idMap[ISchemaEnum.Tag] : idMap[ISchemaEnum.Edge], [type]); return ( @@ -154,21 +152,11 @@ const FileMapping = (props: IProps) => {
{intl.get('import.dataSourceFile')} - - {/* */} + + {selectFile.file &&
+ {intl.get('import.filePath')} + {selectFile.cachedState.path + selectFile.file.name} +
}
onRemove(item)} />
@@ -180,7 +168,11 @@ const FileMapping = (props: IProps) => { rowKey="name" pagination={false} /> - {visible && setVisible(false)} onConfirm={() => {}} />} + {visible && setVisible(false)} + onConfirm={handleUpdateFile} />} ); }; diff --git a/app/pages/Import/TaskCreate/SchemaConfig/index.module.less b/app/pages/Import/TaskCreate/SchemaConfig/index.module.less index 10e9d5bc..31ebfdb4 100644 --- a/app/pages/Import/TaskCreate/SchemaConfig/index.module.less +++ b/app/pages/Import/TaskCreate/SchemaConfig/index.module.less @@ -106,7 +106,28 @@ border-bottom: 1px solid @gray; } .operation { - max-width: 100%; + width: 100%; + .pathRow { + margin: 16px 0; + display: flex; + align-items: center; + } + .pathLabel { + font-weight: 400; + font-size: 14px; + &::after { + content: ':'; + } + } + .pathValue { + flex: 1; + margin-left: 10px; + background: #F3F6F9; + padding: 6px 0 6px 8px; + } + } + .btnClose { + } .fileSelect { :global { @@ -144,6 +165,7 @@ .spaceBetween { display: flex; justify-content: space-between; + align-items: baseline; } :global(.ant-select) { width: fit-content; From c5cbafccad128f5a823c76f0517b2c8f87ac79d7 Mon Sep 17 00:00:00 2001 From: hetao92 <18328704+hetao92@users.noreply.github.com> Date: Mon, 20 Mar 2023 10:47:30 +0000 Subject: [PATCH 07/11] feat: support concat id & edit config --- .../CSVPreviewLink/index.module.less | 4 + app/components/CSVPreviewLink/index.tsx | 63 +++++++--- app/config/locale/en-US.json | 7 +- app/config/locale/zh-CN.json | 7 +- app/config/service.ts | 2 +- app/interfaces/datasource.ts | 9 -- app/interfaces/import.ts | 24 ++++ .../PlatformConfig/S3ConfigForm.tsx | 26 ++++- .../PlatformConfig/SftpConfigForm.tsx | 23 +++- .../DatasourceConfig/PlatformConfig/index.tsx | 49 +++++--- .../DatasourceList/RemoteList/index.tsx | 4 +- .../FileMapping/FileSelectModal.tsx | 14 ++- .../SchemaConfig/FileMapping/index.tsx | 44 ++++++- .../TaskCreate/SchemaConfig/index.module.less | 23 +++- app/stores/datasource.ts | 4 + app/stores/import.ts | 14 ++- app/utils/import.ts | 109 ++++++++++++++---- .../datasource/datasourceupdatehandler.go | 33 ++++++ server/api/studio/internal/handler/routes.go | 5 + .../logic/datasource/datasourceupdatelogic.go | 31 +++++ .../api/studio/internal/service/datasource.go | 69 +++++++++++ server/api/studio/internal/types/types.go | 23 ++++ server/api/studio/restapi/datasource.api | 24 ++++ 23 files changed, 518 insertions(+), 93 deletions(-) create mode 100644 server/api/studio/internal/handler/datasource/datasourceupdatehandler.go create mode 100644 server/api/studio/internal/logic/datasource/datasourceupdatelogic.go diff --git a/app/components/CSVPreviewLink/index.module.less b/app/components/CSVPreviewLink/index.module.less index ca06f7b2..b97b5e31 100644 --- a/app/components/CSVPreviewLink/index.module.less +++ b/app/components/CSVPreviewLink/index.module.less @@ -44,11 +44,15 @@ } > .operation { + display: flex; text-align: center; position: absolute; bottom: 15px; left: 50%; transform: translateX(-50%); + :global(.ant-btn:not(:last-child)) { + margin-right: 23px; + } } .anticon { diff --git a/app/components/CSVPreviewLink/index.tsx b/app/components/CSVPreviewLink/index.tsx index f3f6083d..3791a763 100644 --- a/app/components/CSVPreviewLink/index.tsx +++ b/app/components/CSVPreviewLink/index.tsx @@ -1,26 +1,29 @@ import { Button, Popover, Table } from 'antd'; -import React, { useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { useI18n } from '@vesoft-inc/i18n'; import { v4 as uuidv4 } from 'uuid'; import cls from 'classnames'; import { usePapaParse } from 'react-papaparse'; import { StudioFile } from '@app/interfaces/import'; +import { CheckOutlined } from '@ant-design/icons'; import styles from './index.module.less'; interface IProps { file: StudioFile; children: any; - onMapping?: (index: number) => void; + onMapping?: (index: number[] | number) => void; btnType?: string selected?: boolean + multipleMode?: boolean } const CSVPreviewLink = (props: IProps) => { - const { onMapping, file, children, btnType, selected } = props; + const { onMapping, file, children, btnType, selected, multipleMode } = props; const [visible, setVisible] = useState(false); const [data, setData] = useState([]); const { intl } = useI18n(); const { readString } = usePapaParse(); + const [indexes, setIndexes] = useState([]); useEffect(() => { if(!file) return; const { delimiter, sample } = file; @@ -37,25 +40,48 @@ const CSVPreviewLink = (props: IProps) => { } }); }, [file]); - const handleLinkClick = e => { + const handleLinkClick = useCallback(e => { e.stopPropagation(); setVisible(true); - }; - const handleMapping = (index: number, e: React.MouseEvent) => { + }, []); + const handleMapping = useCallback((e) => { e.stopPropagation(); - onMapping?.(index); + onMapping?.(multipleMode ? indexes : indexes[0]); setVisible(false); - }; + }, [indexes, onMapping]); + const handleClear = useCallback((e) => { + e.stopPropagation(); + onMapping?.(null); + setVisible(false); + setIndexes([]); + }, [onMapping]); + + const toggleMapping = useCallback((index: number, e: React.MouseEvent) => { + e.stopPropagation(); + if(!multipleMode) { + setIndexes([index]); + return; + } + const _indexes = [...indexes]; + if(indexes.indexOf(index) > -1) { + _indexes.splice(indexes.indexOf(index), 1); + } else { + _indexes.push(index); + } + setIndexes(_indexes); + }, [multipleMode, indexes]); + const columns = data[0]?.map((header, index) => { const textIndex = index; const _header = file?.withHeader ? header : `Column ${textIndex}`; + const isSelected = indexes.indexOf(textIndex) > -1; return { title: onMapping ? ( + onClick={(e) => toggleMapping(textIndex, e)} + >{isSelected && }{_header} ) : ( _header ), @@ -63,10 +89,10 @@ const CSVPreviewLink = (props: IProps) => { render: value => {value}, }; }) || []; - const handleOpen = (visible) => { + const handleOpen = useCallback((visible) => { if(!file) return; setVisible(visible); - }; + }, [file]); return ( { />
{onMapping && ( - + <> + + + )}
} diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json index 90e58350..63b7d937 100644 --- a/app/config/locale/en-US.json +++ b/app/config/locale/en-US.json @@ -61,7 +61,8 @@ "src": "Source", "dst": "Destination", "value": "Value", - "continue": "Continue" + "continue": "Continue", + "update": "Update" }, "doc": { "welcome": "Welcome to", @@ -226,11 +227,14 @@ "srcVidColumn": "Source VID column", "dstVidColumn": "Destination VID column", "vidFunction": "VID function", + "vidPrefix": "VID prefix", + "vidSuffix": "VID suffix", "concurrencyTip": "Number of NebulaGraph client concurrency.", "batchSizeTip": "The number of statements inserting data in a batch.", "retryTip": "Retry times of nGQL statement execution failures.", "vidFunctionTip": "Function to generate VID. Currently only hash functions are supported.", "vidPrefixTip": "prefix added to the original vid.", + "vidSuffixTip": "suffix added to the original vid.", "selectCsvColumn": "Select CSV Index", "graphAddress": "Graph service address", "concurrency": "Concurrency", @@ -264,6 +268,7 @@ "s3": "Cloud storage", "sftp": "SFTP", "newDataSource": "New Data Source", + "editDataSource": "Edit Data Source", "deleteDataSource": "Delete Data Source", "datasourceList": "{type} list", "ipAddress": "IP Address:Port", diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json index 5295dc59..83d8f656 100644 --- a/app/config/locale/zh-CN.json +++ b/app/config/locale/zh-CN.json @@ -61,7 +61,8 @@ "src": "起点", "dst": "终点", "value": "值", - "continue": "继续" + "continue": "继续", + "update": "更新" }, "doc": { "welcome": "欢迎使用", @@ -226,11 +227,14 @@ "srcVidColumn": "起点 VID 列", "dstVidColumn": "终点 VID 列", "vidFunction": "VID 函数", + "vidPrefix": "VID 前缀", + "vidSuffix": "VID 后缀", "concurrencyTip": "NebulaGraph 客户端并发数", "batchSizeTip": "单批次插入数据的语句数量", "retryTip": "nGQL 语句执行失败的重试次数", "vidFunctionTip": "生成 VID 的函数。目前只支持 hash 函数", "vidPrefixTip": "给原始 VID 添加的前缀", + "vidSuffixTip": "给原始 VID 添加的后缀", "selectCsvColumn": "选择 CSV 列", "graphAddress": "Graph 服务地址", "concurrency": "并发数", @@ -264,6 +268,7 @@ "s3": "云存储", "sftp": "SFTP", "newDataSource": "新建数据源", + "editDataSource": "编辑数据源", "deleteDataSource": "删除数据源", "datasourceList": "{type}列表", "ipAddress": "IP 地址：端口", diff --git a/app/config/service.ts b/app/config/service.ts index 603e8945..3b30103f 100644 --- a/app/config/service.ts +++ b/app/config/service.ts @@ -100,7 +100,7 @@ const service = { }, updateDatasource: (params, config?) => { const { id, ...restParams } = params; - return put(`/api/datasources/${id}`)(restParams, config); + return post(`/api/datasources/${id}`)(restParams, config); }, deleteDatasource: (id: number, config?) => { return _delete(`/api/datasources/${id}`)(undefined, config); diff --git a/app/interfaces/datasource.ts b/app/interfaces/datasource.ts index 1b73343b..5ed4699a 100644 --- a/app/interfaces/datasource.ts +++ b/app/interfaces/datasource.ts @@ -3,12 +3,3 @@ export enum IDatasourceType { 'sftp' = 'sftp', 'local' = 'local' } - - -export interface ICloudStorage { - ipAddress: string; - bucketName: string; - accessKeyId: string; - region: string; - createTime: number; -} \ No newline at end of file diff --git a/app/interfaces/import.ts b/app/interfaces/import.ts index f16e98e6..23c976e2 100644 --- a/app/interfaces/import.ts +++ b/app/interfaces/import.ts @@ -49,8 +49,32 @@ export interface IImportFile { content: string[]; withHeader?: boolean; delimiter?: string; + s3Config?: IS3Config; + sftpConfig?: ISftpConfig; + /** remote path */ + path?: string; +} + +export interface IS3Config { + region: string; + endpoint: string; + accessKey: string; + accessSecret: string; + bucket?: string; + token?: string; + key: string; } +export interface ISftpConfig { + host: string; + port: number; + username: string; + password: string; + path: string; + keyFile?: string; + keyData?: string; + passPhrase?: string; +} export interface IBasicConfig { taskName: string; address: string[]; diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx index 8998f811..ec23e58a 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx @@ -1,18 +1,32 @@ import { useI18n } from '@vesoft-inc/i18n'; import { Input, Form, Select, FormInstance } from 'antd'; -import React from 'react'; +import React, { useEffect, useState } from 'react'; import { observer } from 'mobx-react-lite'; import styles from './index.module.less'; - const FormItem = Form.Item; interface IProps { - formRef: FormInstance + formRef: FormInstance; + mode: 'create' | 'edit'; + tempPwd: string; } - const S3ConfigForm = (props: IProps) => { - const { formRef } = props; + const { formRef, mode, tempPwd } = props; const { intl } = useI18n(); + const [flag, setFlag] = useState(false); + useEffect(() => { + if(mode === 'edit') { + formRef.setFieldValue(['s3Config', 'accessSecret'], tempPwd); + formRef.setFieldValue('platform', 'aws'); + } + }, [mode]); + const handleUpdatePassword = () => { + if(mode !== 'edit') return; + if(!flag) { + formRef.setFieldValue(['s3Config', 'accessSecret'], ''); + setFlag(true); + } + }; return ( @@ -36,7 +50,7 @@ const S3ConfigForm = (props: IProps) => { - + ); diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/SftpConfigForm.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/SftpConfigForm.tsx index 7883c164..bea64e65 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/SftpConfigForm.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/SftpConfigForm.tsx @@ -1,17 +1,32 @@ import { useI18n } from '@vesoft-inc/i18n'; import { Input, Form, FormInstance } from 'antd'; -import React from 'react'; +import React, { useEffect, useState } from 'react'; import { observer } from 'mobx-react-lite'; const FormItem = Form.Item; interface IProps { - formRef: FormInstance + formRef: FormInstance; + mode: 'create' | 'edit'; + tempPwd: string; } const SftpConfigForm = (props: IProps) => { - const { formRef } = props; + const { formRef, mode, tempPwd } = props; const { intl } = useI18n(); + const [flag, setFlag] = useState(false); + useEffect(() => { + if(mode === 'edit') { + formRef.setFieldValue(['sftpConfig', 'password'], tempPwd); + } + }, [mode]); + const handleUpdatePassword = () => { + if(mode !== 'edit') return; + if(!flag) { + formRef.setFieldValue(['sftpConfig', 'password'], ''); + setFlag(true); + } + }; return ( @@ -24,7 +39,7 @@ const SftpConfigForm = (props: IProps) => { - + ); diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx index b8d2a575..0bbd9294 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx @@ -1,7 +1,8 @@ import { useI18n } from '@vesoft-inc/i18n'; import { Button, Modal, Form, Select, message } from 'antd'; -import React, { useState } from 'react'; +import React, { useMemo, useState } from 'react'; import { IDatasourceType } from '@app/interfaces/datasource'; +import { v4 as uuidv4 } from 'uuid'; import { useStore } from '@app/stores'; import { observer } from 'mobx-react-lite'; import UploadLocalBtn from '../FileUploadBtn'; @@ -26,29 +27,49 @@ const fomrItemLayout = { const DatasourceConfigModal = (props: IProps) => { const { visible, type, onCancel, onConfirm, data } = props; const { datasource } = useStore(); - const { addDataSource } = datasource; + const { addDataSource, updateDataSource } = datasource; const { intl } = useI18n(); const [form] = Form.useForm(); const [loading, setLoading] = useState(false); - + const tempPwd = useMemo(() => uuidv4() + Date.now(), []); + const mode = useMemo(() => (data ? 'edit' : 'create'), [data]); const submit = async (values: any) => { const { platform, ...rest } = values; + const _type = values.type || type; setLoading(true); - const flag = await addDataSource({ - type: values.type, - name: '', - ...rest - }); - setLoading(false); - flag && (message.success(intl.get('schema.createSuccess')), onConfirm()); + if(mode === 'create') { + const flag = await addDataSource({ + type: _type, + name: '', + ...rest + }); + setLoading(false); + flag && (message.success(intl.get('schema.createSuccess')), onConfirm()); + } else { + const _config = _type === IDatasourceType.s3 ? 's3Config' : 'sftpConfig'; + if(_type === IDatasourceType.s3 && rest[_config].accessSecret === tempPwd) { + delete rest[_config].accessSecret; + } else if (_type === IDatasourceType.sftp && rest[_config].password === tempPwd) { + delete rest[_config].password; + } + const flag = await updateDataSource({ + id: data.id, + type: _type, + name: '', + ...rest + }); + setLoading(false); + flag && (message.success(intl.get('common.updateSuccess')), onConfirm()); + } }; return ( @@ -74,9 +95,9 @@ const DatasourceConfigModal = (props: IProps) => { const configType = type || getFieldValue('type'); switch (configType) { case IDatasourceType.s3: - return ; + return ; case IDatasourceType.sftp: - return ; + return ; default: return null; } @@ -96,7 +117,7 @@ const DatasourceConfigModal = (props: IProps) => { } else { return
; } diff --git a/app/pages/Import/DatasourceList/RemoteList/index.tsx b/app/pages/Import/DatasourceList/RemoteList/index.tsx index 8e10ec41..c476daf5 100644 --- a/app/pages/Import/DatasourceList/RemoteList/index.tsx +++ b/app/pages/Import/DatasourceList/RemoteList/index.tsx @@ -62,7 +62,7 @@ const DatasourceList = (props: IProps) => { const { type } = props; const { datasource } = useStore(); const { intl } = useI18n(); - const { getDatasourceList, datasourceList, getDatasourceDetail, previewFile, deleteDataSource, batchDeleteDatasource } = datasource; + const { getDatasourceList, datasourceList, deleteDataSource, batchDeleteDatasource } = datasource; const [data, setData] = useState([]); const [editData, setEditData] = useState(null); const [loading, setLoading] = useState(false); @@ -85,7 +85,7 @@ const DatasourceList = (props: IProps) => { title: intl.get('common.operation'), key: 'operation', render: (_, item) => (
- { init(); }, []); const handleSelectFile = useCallback(async (item) => { + if (item.type !== 'directory') return; setState({ loading: true }); const newPath = `${path === '/' ? '' : path}${item.name}${item.type === 'directory' ? '/' : ''}`; - if (item.type !== 'directory') return; - if(item.type === 'directory') { - getDatasourceDirectory(activeId, newPath); - } + getDatasourceDirectory(activeId, newPath); }, [path]); const handlePathBack = useCallback(async () => { @@ -119,7 +117,13 @@ const FileSelect = observer((props: IFileSelect) => { withHeader: false, delimiter: ',', sample: data.contents.join('\r\n'), - }; + path: _path, + } as any; + if(activeId !== IDatasourceType.local) { + const { sftpConfig, s3Config } = options.find((item) => item.id === activeId); + item.sftpConfig = sftpConfig; + item.s3Config = s3Config; + } setState({ loading: false }); onConfirm(item, state); } diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx index 43edaf11..4b15d9a3 100644 --- a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx +++ b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx @@ -1,4 +1,4 @@ -import { Button, Collapse, Select, Table, Tooltip } from 'antd'; +import { Button, Collapse, Input, Select, Table, Tooltip } from 'antd'; import React, { useMemo, useState } from 'react'; import { observer } from 'mobx-react-lite'; import cls from 'classnames'; @@ -30,10 +30,11 @@ const VIDSetting = observer((props: { idKey: string, idFunction?: string, idPrefix?: string, + idSuffix?: string, label: string } }) => { - const { keyMap: { idKey, idFunction, idPrefix, label }, data } = props; + const { keyMap: { idKey, idFunction, idPrefix, idSuffix, label }, data } = props; const { intl } = useI18n(); const { schema } = useStore(); const { spaceVidType } = schema; @@ -45,8 +46,9 @@ const VIDSetting = observer((props: { data.update({ [idKey]: index })} file={data.file} + multipleMode={true} > - {!data[idKey] && data[idKey] !== 0 ? intl.get('import.selectCsvColumn') : `Column ${data[idKey]}`} + {(!data[idKey] || data[idKey].length === 0) ? intl.get('import.selectCsvColumn') : `Column ${data[idKey]}`}
@@ -54,6 +56,14 @@ const VIDSetting = observer((props: { {intl.get('import.vidFunction')} {intl.get('import.vidFunctionTip')} +
+ {intl.get('import.vidPrefix')} + {intl.get('import.vidPrefixTip')} +
+
+ {intl.get('import.vidSuffix')} + {intl.get('import.vidSuffixTip')} +
} /> } key="default"> {spaceVidType === 'INT64' && idFunction &&
@@ -72,6 +82,22 @@ const VIDSetting = observer((props: {
} + {idPrefix &&
+ {intl.get('import.vidPrefix')} + data.update({ [idPrefix]: e.target.value })} /> +
} + {idSuffix &&
+ {intl.get('import.vidSuffix')} + data.update({ [idSuffix]: e.target.value })} /> +
} + {(data[idKey]?.length > 1 || data[idPrefix] || data[idSuffix]) &&
+ {intl.get('import.preview')} +
+ {data[idPrefix] && {data[idPrefix]}} + {data[idKey].map(i => {`Column ${i}`})} + {data[idSuffix] && {data[idSuffix]}} +
+
} ; @@ -81,16 +107,22 @@ const idMap = { [ISchemaEnum.Tag]: [{ idKey: 'vidIndex', idFunction: 'vidFunction', + idPrefix: 'vidPrefix', + idSuffix: 'vidSuffix', label: 'vidColumn' }], [ISchemaEnum.Edge]: [{ idKey: 'srcIdIndex', idFunction: 'srcIdFunction', - label: 'srcVidColumn' + label: 'srcVidColumn', + idPrefix: 'srcIdPrefix', + idSuffix: 'srcIdSuffix', }, { idKey: 'dstIdIndex', idFunction: 'dstIdFunction', - label: 'dstVidColumn' + label: 'dstVidColumn', + idPrefix: 'dstIdPrefix', + idSuffix: 'dstIdSuffix', }], }; @@ -127,7 +159,7 @@ const FileMapping = (props: IProps) => { dataIndex: 'mapping', render: (mappingIndex, _, propIndex) => ( updateFilePropMapping(propIndex, columnIndex)} + onMapping={columnIndex => updateFilePropMapping(propIndex, columnIndex as number)} file={file} > {!mappingIndex && mappingIndex !== 0 ? intl.get('import.choose') : `Column ${mappingIndex}`} diff --git a/app/pages/Import/TaskCreate/SchemaConfig/index.module.less b/app/pages/Import/TaskCreate/SchemaConfig/index.module.less index 31ebfdb4..aec3eafa 100644 --- a/app/pages/Import/TaskCreate/SchemaConfig/index.module.less +++ b/app/pages/Import/TaskCreate/SchemaConfig/index.module.less @@ -126,9 +126,7 @@ padding: 6px 0 6px 8px; } } - .btnClose { - - } + .fileSelect { :global { .ant-select-selection-placeholder, .anticon, .ant-select-selection-item { @@ -143,6 +141,25 @@ width: 45%; display: inline-block; } + .concatPreview { + display: flex; + margin-top: 15px; + align-items: center; + } + .concatItems { + background: #F3F6F9; + padding: 8px; + flex: 1; + .tagItem { + background: #8697B0; + border-radius: 3px; + padding: 6px; + color: #fff; + &:not(:last-child) { + margin-right: 10px; + } + } + } .functionSelect { width: 60%; background: @lightBlue; diff --git a/app/stores/datasource.ts b/app/stores/datasource.ts index caa3a05a..9c7f4536 100644 --- a/app/stores/datasource.ts +++ b/app/stores/datasource.ts @@ -22,6 +22,10 @@ export class DatasourceStore { const { code } = await service.addDatasource(payload); return code === 0; }; + updateDataSource = async (payload) => { + const { code } = await service.updateDatasource(payload); + return code === 0; + }; getDatasourceList = async (payload?: { type?: string }) => { const { code, data } = await service.getDatasourceList(payload); if(code === 0) { diff --git a/app/stores/import.ts b/app/stores/import.ts index cf581e09..5e4dd44a 100644 --- a/app/stores/import.ts +++ b/app/stores/import.ts @@ -21,9 +21,10 @@ const handlePropertyMap = (item, defaultValueFields) => { export class TagFileItem { file: IImportFile; props = observable.array([]); - vidIndex?: number; + vidIndex = observable.array([]); vidFunction?: string; - + vidPrefix?: string; + vidSuffix?: string; constructor({ file, props }: { file?: IImportFile; props?: IPropertyProps[] }) { makeAutoObservable(this); file && (this.file = file); @@ -44,11 +45,14 @@ export class TagFileItem { export class EdgeFileItem { file: IImportFile; props = observable.array([]); - srcIdIndex?: number; - dstIdIndex?: number; + srcIdIndex = observable.array([]); + dstIdIndex = observable.array([]); srcIdFunction?: string; dstIdFunction?: string; - + srcIdPrefix?: string; + dstIdPrefix?: string; + srcIdSuffix?: string; + dstIdSuffix?: string; constructor({ file, props }: { file?: IImportFile; props?: IPropertyProps[] }) { makeAutoObservable(this); file && (this.file = file); diff --git a/app/utils/import.ts b/app/utils/import.ts index d7ed8e68..1a4f292c 100644 --- a/app/utils/import.ts +++ b/app/utils/import.ts @@ -56,6 +56,25 @@ export function configToJson(payload: IConfig) { return configJson; } +const getIdConfig = (payload: { + indexes, + prefix, + suffix, + vidFunction, + type +}) => { + const { indexes, prefix, suffix, vidFunction, type } = payload; + const id = { + type, + function: vidFunction, + } as any; + if(indexes.length > 1 || !!prefix || !!suffix) { + id.concatItems = [prefix, ...indexes, suffix]; + } else { + id.index = indexes[0]; + } + return id; +}; export function edgeDataToJSON( configs: IEdgeItem[], spaceVidType: string, @@ -63,7 +82,7 @@ export function edgeDataToJSON( const result = configs.reduce((acc: any, cur) => { const { name, files } = cur; const _config = files.map(item => { - const { file, props, srcIdIndex, srcIdFunction, dstIdIndex, dstIdFunction } = item; + const { file, props, srcIdIndex, srcIdFunction, dstIdIndex, dstIdFunction, srcIdPrefix, srcIdSuffix, dstIdPrefix, dstIdSuffix } = item; const vidType = spaceVidType === 'INT64' ? 'int' : 'string'; // rank is the last prop const rank = props[props.length - 1]; @@ -81,30 +100,56 @@ export function edgeDataToJSON( const edges = [{ name: handleEscape(name), src: { - id: { + id: getIdConfig({ + indexes: srcIdIndex, + prefix: srcIdPrefix, + suffix: srcIdSuffix, + vidFunction: srcIdFunction, type: vidType, - index: srcIdIndex, - function: srcIdFunction, - } + }) }, dst: { - id: { + id: getIdConfig({ + indexes: dstIdIndex, + prefix: dstIdPrefix, + suffix: dstIdSuffix, + vidFunction: dstIdFunction, type: vidType, - index: dstIdIndex, - function: dstIdFunction, - } + }) }, rank: typeof rank.mapping == 'number' ? { index: rank.mapping } : null, props: edgeProps, }]; const edgeConfig = { - path: file.name, csv: { withHeader: file.withHeader || false, delimiter: file.delimiter }, edges, - }; + } as any; + const { sftpConfig, s3Config } = file; + if(s3Config) { + const { accessKey, accessSecret, bucket, endpoint, region } = s3Config; + edgeConfig.s3 = { + accessKey, + secretKey: accessSecret, + endpoint, + bucket, + region, + key: file.path + }; + } else if(sftpConfig) { + const { host, port, username, password } = sftpConfig; + edgeConfig.sftp = { + host, + port, + username, + password, + path: file.path + }; + } else { + edgeConfig.path = file.name; + } return edgeConfig; }); acc.push(..._config); @@ -120,7 +165,7 @@ export function tagDataToJSON( const result = configs.reduce((acc: any, cur) => { const { name, files } = cur; const _config = files.map(item => { - const { file, props, vidIndex, vidFunction } = item; + const { file, props, vidIndex, vidFunction, vidPrefix, vidSuffix } = item; const _props = props.reduce((acc: any, cur) => { if (isEmpty(cur.mapping) && (cur.allowNull || cur.isDefault)) { return acc; @@ -132,24 +177,48 @@ export function tagDataToJSON( }); return acc; }, []); - const tags = [{ name: handleEscape(name), - id: { + id: getIdConfig({ + indexes: vidIndex, + prefix: vidPrefix, + suffix: vidSuffix, + vidFunction, type: spaceVidType === 'INT64' ? 'int' : 'string', - index: vidIndex, - function: vidFunction, - }, + }), props: _props.filter(prop => prop), }]; - return { - path: file.name, + const result = { csv: { withHeader: file.withHeader || false, delimiter: file.delimiter }, tags - }; + } as any; + const { sftpConfig, s3Config } = file; + if(s3Config) { + const { accessKey, accessSecret, bucket, endpoint, region } = s3Config; + result.s3 = { + accessKey, + secretKey: accessSecret, + endpoint, + bucket, + region, + key: file.path + }; + } else if(sftpConfig) { + const { host, port, username, password } = sftpConfig; + result.sftp = { + host, + port, + username, + password, + path: file.path + }; + } else { + result.path = file.name; + } + return result; }); acc.push(..._config); return acc; diff --git a/server/api/studio/internal/handler/datasource/datasourceupdatehandler.go b/server/api/studio/internal/handler/datasource/datasourceupdatehandler.go new file mode 100644 index 00000000..8d4f938b --- /dev/null +++ b/server/api/studio/internal/handler/datasource/datasourceupdatehandler.go @@ -0,0 +1,33 @@ +// Code generated by goctl. DO NOT EDIT. +package datasource + +import ( + "net/http" + + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/datasource" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" +) + +func DatasourceUpdateHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.DatasourceUpdateRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithErrorMessage(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + + l := datasource.NewDatasourceUpdateLogic(r.Context(), svcCtx) + err := l.DatasourceUpdate(req) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + } +} diff --git a/server/api/studio/internal/handler/routes.go b/server/api/studio/internal/handler/routes.go index cdd01ed1..4b54b7fb 100644 --- a/server/api/studio/internal/handler/routes.go +++ b/server/api/studio/internal/handler/routes.go @@ -212,6 +212,11 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { Path: "/api/datasources", Handler: datasource.DatasourceAddHandler(serverCtx), }, + { + Method: http.MethodPost, + Path: "/api/datasources/:id", + Handler: datasource.DatasourceUpdateHandler(serverCtx), + }, { Method: http.MethodDelete, Path: "/api/datasources/:id", diff --git a/server/api/studio/internal/logic/datasource/datasourceupdatelogic.go b/server/api/studio/internal/logic/datasource/datasourceupdatelogic.go new file mode 100644 index 00000000..1d7208d9 --- /dev/null +++ b/server/api/studio/internal/logic/datasource/datasourceupdatelogic.go @@ -0,0 +1,31 @@ +package datasource + +import ( + "context" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type DatasourceUpdateLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewDatasourceUpdateLogic(ctx context.Context, svcCtx *svc.ServiceContext) DatasourceUpdateLogic { + return DatasourceUpdateLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *DatasourceUpdateLogic) DatasourceUpdate(req types.DatasourceUpdateRequest) error { + return service.NewDatasourceService(l.ctx, l.svcCtx).Update(req) + +} diff --git a/server/api/studio/internal/service/datasource.go b/server/api/studio/internal/service/datasource.go index d9696263..13c48097 100644 --- a/server/api/studio/internal/service/datasource.go +++ b/server/api/studio/internal/service/datasource.go @@ -28,6 +28,7 @@ import ( type ( DatasourceService interface { Add(request types.DatasourceAddRequest) (*types.DatasourceAddData, error) + Update(request types.DatasourceUpdateRequest) error List(request types.DatasourceListRequest) (*types.DatasourceData, error) Remove(request types.DatasourceRemoveRequest) error BatchRemove(request types.DatasourceBatchRemoveRequest) error @@ -100,6 +101,58 @@ func (d *datasourceService) Add(request types.DatasourceAddRequest) (*types.Data return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, nil, "datasource type can't support'") } +func (d *datasourceService) Update(request types.DatasourceUpdateRequest) error { + datasourceId := request.ID + dbs, err := d.findOne(datasourceId) + if err != nil { + return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "find data error") + } + switch request.Type { + case "s3": + c := request.S3Config + if c.AccessSecret == "" { + c.AccessSecret = dbs.Secret + } + if err := d.testConnectionS3(c.Endpoint, c.Region, c.Bucket, c.AccessKey, c.AccessSecret); err != nil { + return err + } + secret := c.AccessSecret + c.AccessSecret = "" + cstr, err := json.Marshal(c) + if err != nil { + return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "json stringify error") + } + crypto, err := utils.Encrypt([]byte(secret), []byte(cipher)) + err = d.update(datasourceId, request.Type, request.Name, string(cstr), crypto) + if err != nil { + return err + } + return nil + case "sftp": + c := request.SFTPConfig + if c.Password == "" { + c.Password = dbs.Secret + } + if err := d.testConnectionSFTP(c.Host, c.Port, c.Username, c.Password); err != nil { + return err + } + pwd := c.Password + c.Password = "" + cstr, err := json.Marshal(c) + if err != nil { + d.Logger.Errorf("json stringify error", c) + return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "json stringify error") + } + crypto, err := utils.Encrypt([]byte(pwd), []byte(cipher)) + err = d.update(datasourceId, request.Type, request.Name, string(cstr), crypto) + if err != nil { + return err + } + return nil + } + return ecode.WithErrorMessage(ecode.ErrBadRequest, nil, "datasource type can't support'") +} + func (d *datasourceService) List(request types.DatasourceListRequest) (*types.DatasourceData, error) { user := d.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) host := user.Address + ":" + strconv.Itoa(user.Port) @@ -260,6 +313,22 @@ func (d *datasourceService) save(typ, name, config, secret string) (id int, err } return int(dbs.ID), nil } +func (d *datasourceService) update(id int, typ, name, config, secret string) (err error) { + user := d.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) + host := user.Address + ":" + strconv.Itoa(user.Port) + result := db.CtxDB.Model(&db.Datasource{ID: id}).Updates(map[string]interface{}{ + "type": typ, + "name": name, + "config": config, + "secret": secret, + "host": host, + "username": user.Username, + }) + if result.Error != nil { + return d.gormErrorWrapper(result.Error) + } + return nil +} // TODO: cache the store connection to improve the request handle speed by the go-zero session func (d *datasourceService) getFileStore(dbs *db.Datasource) (filestore.FileStore, error) { diff --git a/server/api/studio/internal/types/types.go b/server/api/studio/internal/types/types.go index 197ca8d2..ab44c6b2 100644 --- a/server/api/studio/internal/types/types.go +++ b/server/api/studio/internal/types/types.go @@ -356,6 +356,21 @@ type DatasourceSFTPConfig struct { Password string `json:"password"` } +type DatasourceS3UpdateConfig struct { + Endpoint string `json:"endpoint, optional, omitempty"` + Region string `json:"region, optional, omitempty"` + Bucket string `json:"bucket, optional, omitempty"` + AccessKey string `json:"accessKey, optional, omitempty"` + AccessSecret string `json:"accessSecret, optional, omitempty"` +} + +type DatasourceSFTPUpdateConfig struct { + Host string `json:"host, optional, omitempty"` + Port int `json:"port, optional, omitempty"` + Username string `json:"username, optional, omitempty"` + Password string `json:"password, optional, omitempty"` +} + type DatasourceAddRequest struct { Type string `json:"type"` Name string `json:"name"` @@ -363,6 +378,14 @@ type DatasourceAddRequest struct { SFTPConfig *DatasourceSFTPConfig `json:"sftpConfig,optional"` } +type DatasourceUpdateRequest struct { + ID int `path:"id"` + Type string `json:"type"` + Name string `json:"name"` + S3Config *DatasourceS3UpdateConfig `json:"s3Config,optional"` + SFTPConfig *DatasourceSFTPUpdateConfig `json:"sftpConfig,optional"` +} + type DatasourceAddData struct { ID int `json:"id"` } diff --git a/server/api/studio/restapi/datasource.api b/server/api/studio/restapi/datasource.api index 5e7442ab..d77e8aba 100644 --- a/server/api/studio/restapi/datasource.api +++ b/server/api/studio/restapi/datasource.api @@ -15,6 +15,20 @@ type ( Username string `json:"username"` Password string `json:"password"` } + DatasourceS3UpdateConfig { + Endpoint string `json:"endpoint, optional, omitempty"` + Region string `json:"region, optional, omitempty"` + Bucket string `json:"bucket, optional, omitempty"` + AccessKey string `json:"accessKey, optional, omitempty"` + AccessSecret string `json:"accessSecret, optional, omitempty"` + } + + DatasourceSFTPUpdateConfig { + Host string `json:"host, optional, omitempty"` + Port int `json:"port, optional, omitempty"` + Username string `json:"username, optional, omitempty"` + Password string `json:"password, optional, omitempty"` + } DatasourceAddRequest { Type string `json:"type"` @@ -22,6 +36,13 @@ type ( S3Config *DatasourceS3Config `json:"s3Config,optional"` SFTPConfig *DatasourceSFTPConfig `json:"sftpConfig,optional"` } + DatasourceUpdateRequest { + ID int `path:"id"` + Type string `json:"type"` + Name string `json:"name"` + S3Config *DatasourceS3UpdateConfig `json:"s3Config,optional"` + SFTPConfig *DatasourceSFTPUpdateConfig `json:"sftpConfig,optional"` + } DatasourceAddData { ID int `json:"id"` @@ -84,6 +105,9 @@ service studio-api { @doc "Add Datasource" @handler DatasourceAdd post /api/datasources(DatasourceAddRequest) returns(DatasourceAddData) + @doc "Update Datasource" + @handler DatasourceUpdate + post /api/datasources/:id(DatasourceUpdateRequest) @doc "Remove Datasource" @handler DatasourceRemove From 67078514bbeba79361c97fd949b19aa731130eee Mon Sep 17 00:00:00 2001 From: hetao92 <18328704+hetao92@users.noreply.github.com> Date: Tue, 21 Mar 2023 10:24:49 +0000 Subject: [PATCH 08/11] feat: get config from db before import & fix some issues --- app/components/CSVPreviewLink/index.tsx | 17 +- app/interfaces/import.ts | 4 +- .../PlatformConfig/S3ConfigForm.tsx | 5 +- .../PlatformConfig/SftpConfigForm.tsx | 4 +- .../TaskCreate/FileSelect/index.module.less | 11 -- .../Import/TaskCreate/FileSelect/index.tsx | 86 --------- .../SchemaConfig/FileMapping/FileSelect.tsx | 177 +++++++++++++++++ .../FileMapping/FileSelectModal.tsx | 179 +----------------- .../SchemaConfig/FileMapping/index.tsx | 2 + app/stores/import.ts | 4 +- app/utils/import.ts | 56 ++---- .../api/studio/internal/service/datasource.go | 66 ++----- server/api/studio/internal/service/import.go | 70 ++++++- .../internal/service/importer/taskmgr.go | 19 +- server/api/studio/internal/types/types.go | 18 +- .../api/studio/pkg/utils/datasourceParse.go | 49 +++++ server/api/studio/restapi/import.api | 18 +- 17 files changed, 379 insertions(+), 406 deletions(-) delete mode 100644 app/pages/Import/TaskCreate/FileSelect/index.module.less delete mode 100644 app/pages/Import/TaskCreate/FileSelect/index.tsx create mode 100644 app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelect.tsx create mode 100644 server/api/studio/pkg/utils/datasourceParse.go diff --git a/app/components/CSVPreviewLink/index.tsx b/app/components/CSVPreviewLink/index.tsx index 3791a763..4264dbf3 100644 --- a/app/components/CSVPreviewLink/index.tsx +++ b/app/components/CSVPreviewLink/index.tsx @@ -14,13 +14,14 @@ interface IProps { onMapping?: (index: number[] | number) => void; btnType?: string selected?: boolean - multipleMode?: boolean + multipleMode?: boolean, + data: number[] | number } const CSVPreviewLink = (props: IProps) => { - const { onMapping, file, children, btnType, selected, multipleMode } = props; + const { onMapping, file, children, btnType, selected, multipleMode, data } = props; const [visible, setVisible] = useState(false); - const [data, setData] = useState([]); + const [datasource, setDatasource] = useState([]); const { intl } = useI18n(); const { readString } = usePapaParse(); const [indexes, setIndexes] = useState([]); @@ -36,10 +37,14 @@ const CSVPreviewLink = (props: IProps) => { data = [...data, row.data]; }, complete: () => { - setData(data); + setDatasource(data); } }); }, [file]); + + useEffect(() => { + setIndexes(data ? (Array.isArray(data) ? data : [data]) : []); + }, [data]); const handleLinkClick = useCallback(e => { e.stopPropagation(); setVisible(true); @@ -71,7 +76,7 @@ const CSVPreviewLink = (props: IProps) => { setIndexes(_indexes); }, [multipleMode, indexes]); - const columns = data[0]?.map((header, index) => { + const columns = datasource[0]?.map((header, index) => { const textIndex = index; const _header = file?.withHeader ? header : `Column ${textIndex}`; const isSelected = indexes.indexOf(textIndex) > -1; @@ -106,7 +111,7 @@ const CSVPreviewLink = (props: IProps) => {
uuidv4()} diff --git a/app/interfaces/import.ts b/app/interfaces/import.ts index 23c976e2..7ef3de23 100644 --- a/app/interfaces/import.ts +++ b/app/interfaces/import.ts @@ -51,6 +51,8 @@ export interface IImportFile { delimiter?: string; s3Config?: IS3Config; sftpConfig?: ISftpConfig; + /** remote config id */ + datasourceId?: number; /** remote path */ path?: string; } @@ -60,7 +62,7 @@ export interface IS3Config { endpoint: string; accessKey: string; accessSecret: string; - bucket?: string; + bucket: string; token?: string; key: string; } diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx index ec23e58a..22ebae53 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx @@ -33,14 +33,15 @@ const S3ConfigForm = (props: IProps) => { - + diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/SftpConfigForm.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/SftpConfigForm.tsx index bea64e65..a7956b1c 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/SftpConfigForm.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/SftpConfigForm.tsx @@ -1,5 +1,5 @@ import { useI18n } from '@vesoft-inc/i18n'; -import { Input, Form, FormInstance } from 'antd'; +import { Input, Form, FormInstance, InputNumber } from 'antd'; import React, { useEffect, useState } from 'react'; import { observer } from 'mobx-react-lite'; @@ -33,7 +33,7 @@ const SftpConfigForm = (props: IProps) => { - + diff --git a/app/pages/Import/TaskCreate/FileSelect/index.module.less b/app/pages/Import/TaskCreate/FileSelect/index.module.less deleted file mode 100644 index bc5f3285..00000000 --- a/app/pages/Import/TaskCreate/FileSelect/index.module.less +++ /dev/null @@ -1,11 +0,0 @@ -.popoverFileSelect { - margin-bottom: 20px; - :global(.ant-popover-title) { - border-bottom: none; - } -} -.fileSelectForm { - .fileSelect { - width: 280px; - } -} diff --git a/app/pages/Import/TaskCreate/FileSelect/index.tsx b/app/pages/Import/TaskCreate/FileSelect/index.tsx deleted file mode 100644 index 9cfad631..00000000 --- a/app/pages/Import/TaskCreate/FileSelect/index.tsx +++ /dev/null @@ -1,86 +0,0 @@ -import { Button, Form, Popover, Select } from 'antd'; -import React, { useState } from 'react'; -import { observer } from 'mobx-react-lite'; -import { useStore } from '@app/stores'; -import { v4 as uuidv4 } from 'uuid'; -import Icon from '@app/components/Icon'; -import { useI18n } from '@vesoft-inc/i18n'; -import styles from './index.module.less'; - -const Option = Select.Option; - -interface IProps { - type: 'vertices' | 'edge'; -} -const FormItem = Form.Item; - -const FileSelect = (props: IProps) => { - const { type } = props; - const { files, dataImport: { update, verticesConfig, edgeConfig } } = useStore(); - const { fileList, getFiles } = files; - const [visible, setVisible] = useState(false); - const { intl } = useI18n(); - const onFinish = (value) => { - const file = fileList.filter(item => item.name === value.name)[0]; - if(type === 'vertices') { - update({ - verticesConfig: [...verticesConfig, { - name: uuidv4(), - file, - tags: [], - idMapping: null, - }] - }); - } else { - update({ - edgeConfig: [...edgeConfig, { - name: uuidv4(), - file, - props: [], - type: '', - }] - }); - } - setVisible(false); - }; - - const handleGetFiles = () => { - if(fileList.length === 0) { - getFiles(); - } - }; - - return ( - setVisible(visible)} - content={ - - - - - - - } - title={intl.get('import.selectFile')} - > - - - ); -}; - -export default observer(FileSelect); diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelect.tsx b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelect.tsx new file mode 100644 index 00000000..09b724ec --- /dev/null +++ b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelect.tsx @@ -0,0 +1,177 @@ +import { ArrowLeftOutlined, FileTextFilled, FolderFilled, SyncOutlined } from '@ant-design/icons'; +import { IDatasourceType } from '@app/interfaces/datasource'; +import { useStore } from '@app/stores'; +import { useBatchState } from '@app/utils'; +import { getFileSize } from '@app/utils/file'; +import { useI18n } from '@vesoft-inc/i18n'; +import { Button, Select, Spin } from 'antd'; +import cls from 'classnames'; +import { observer } from 'mobx-react-lite'; +import React, { useCallback, useEffect } from 'react'; +import styles from './index.module.less'; +const Option = Select.Option; +interface IFileSelect { + onConfirm: (file, cachedState) => void, + cachedState?: any +} +const FileSelect = observer((props: IFileSelect) => { + const { intl } = useI18n(); + const { datasource, files } = useStore(); + const { onConfirm, cachedState } = props; + const { getDatasourceList, getDatasourceDetail, previewFile } = datasource; + const { getFiles } = files; + const { state, setState } = useBatchState({ + loading: false, + options: [], + directory: cachedState?.directory || [], + path: cachedState?.path || '', + activeId: cachedState?.activeId, + activeItem: cachedState?.activeItem, + }); + const { options, directory, path, activeItem, activeId, loading } = state; + const init = useCallback(async () => { + setState({ loading: true }); + const data = await getDatasourceList(); + setState({ + options: data, + loading: false, + }); + }, []); + + const getLocalFiles = useCallback(async () => { + const files = await getFiles(); + setState({ + directory: files, + path: '/', + loading: false, + activeId: IDatasourceType.local + }); + }, []); + const getDatasourceDirectory = useCallback(async (id, path?) => { + const data = await getDatasourceDetail({ id, path }); + setState({ + directory: data, + activeId: id, + path: path || '/', + loading: false + }); + }, []); + + useEffect(() => { + init(); + }, []); + const handleSelectFile = useCallback(async (item) => { + if (item.type !== 'directory') return; + setState({ loading: true }); + const newPath = `${path === '/' ? '' : path}${item.name}${item.type === 'directory' ? '/' : ''}`; + getDatasourceDirectory(activeId, newPath); + }, [path]); + + const handlePathBack = useCallback(async () => { + if(!path || path === '/') return; + setState({ loading: true }); + const _path = path.slice(0, -1).split('/'); + _path.pop(); + const newPath = _path.join('/').length ? _path.join('/') + '/' : ''; + getDatasourceDirectory(activeId, newPath); + }, [path, activeId]); + const handleTypeChange = useCallback(async (value) => { + setState({ + loading: true, + activeId: null, + activeItem: null, + path: '', + }); + if(value === IDatasourceType.local) { + getLocalFiles(); + } else { + getDatasourceDirectory(value); + } + }, []); + const handleRefresh = useCallback(async () => { + setState({ loading: true }); + if (!activeId) { + getLocalFiles(); + return; + } + const _path = !path || path === '/' ? '' : path; + getDatasourceDirectory(activeId, _path); + }, [activeId, path]); + const handleConfirm = useCallback(async () => { + if(activeItem && activeId === IDatasourceType.local) { + // select local file + onConfirm(activeItem, state); + } else { + setState({ loading: true }); + const _path = `${path === '/' ? '' : path}${activeItem.name}`; + const data = await previewFile({ id: activeId, path: _path }); + const item = { + name: activeItem.name, + withHeader: false, + delimiter: ',', + sample: data.contents.join('\r\n'), + path: _path, + datasourceId: activeId === IDatasourceType.local ? null : activeId, + } as any; + setState({ loading: false }); + onConfirm(item, state); + } + }, [activeItem, activeId, path]); + return +
+ {intl.get('import.datasourceType')} + +
+
+ {intl.get('import.filePath')} +
+
{path}
+
+
+
+
+
+ {directory.map((item) => ( +
setState({ activeItem: item })} + onDoubleClick={() => handleSelectFile(item)} + > + {item.type === 'directory' ? : } +
+ {item.name} + {item.type === 'directory' ? intl.get('import.directory') : getFileSize(item.size)} +
+
+ ))} +
+
+ +
+ ; +}); + +export default FileSelect; \ No newline at end of file diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelectModal.tsx b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelectModal.tsx index a7f0afb2..f525f22f 100644 --- a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelectModal.tsx +++ b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelectModal.tsx @@ -1,189 +1,16 @@ -import { ArrowLeftOutlined, FileTextFilled, FolderFilled, SyncOutlined } from '@ant-design/icons'; -import { IDatasourceType } from '@app/interfaces/datasource'; -import { useStore } from '@app/stores'; -import { useBatchState } from '@app/utils'; -import { getFileSize } from '@app/utils/file'; import { useI18n } from '@vesoft-inc/i18n'; -import { Button, Modal, Select, Spin } from 'antd'; -import cls from 'classnames'; +import { Modal } from 'antd'; import { observer } from 'mobx-react-lite'; -import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import React, { useMemo, useState } from 'react'; import FileConfigSetting from '@app/components/FileConfigSetting'; +import FileSelect from './FileSelect'; import styles from './index.module.less'; -const Option = Select.Option; interface IModalProps { visible: boolean; onConfirm: (file, cachedDatasourceState) => void onCancel: () => void; cachedDatasourceState?: any; } -interface IFileSelect { - onConfirm: (file, cachedState) => void, - cachedState?: any -} -const FileSelect = observer((props: IFileSelect) => { - const { intl } = useI18n(); - const { datasource, files } = useStore(); - const { onConfirm, cachedState } = props; - const { getDatasourceList, getDatasourceDetail, previewFile } = datasource; - const { getFiles } = files; - const { state, setState } = useBatchState({ - loading: false, - options: [], - directory: cachedState?.directory || [], - path: cachedState?.path || '', - activeId: cachedState?.activeId, - activeItem: cachedState?.activeItem, - }); - const { options, directory, path, activeItem, activeId, loading } = state; - const init = useCallback(async () => { - setState({ loading: true }); - const data = await getDatasourceList(); - setState({ - options: data, - loading: false, - }); - }, []); - - const getLocalFiles = useCallback(async () => { - const files = await getFiles(); - setState({ - directory: files, - path: '/', - loading: false, - activeId: IDatasourceType.local - }); - }, []); - const getDatasourceDirectory = useCallback(async (id, path?) => { - const data = await getDatasourceDetail({ id, path }); - setState({ - directory: data, - activeId: id, - path: path || '/', - loading: false - }); - }, []); - - useEffect(() => { - init(); - }, []); - const handleSelectFile = useCallback(async (item) => { - if (item.type !== 'directory') return; - setState({ loading: true }); - const newPath = `${path === '/' ? '' : path}${item.name}${item.type === 'directory' ? '/' : ''}`; - getDatasourceDirectory(activeId, newPath); - }, [path]); - - const handlePathBack = useCallback(async () => { - if(!path || path === '/') return; - setState({ loading: true }); - const _path = path.slice(0, -1).split('/'); - _path.pop(); - const newPath = _path.join('/').length ? _path.join('/') + '/' : ''; - getDatasourceDirectory(activeId, newPath); - }, [path, activeId]); - const handleTypeChange = useCallback(async (value) => { - setState({ - loading: true, - activeId: null, - activeItem: null, - path: '', - }); - if(value === IDatasourceType.local) { - getLocalFiles(); - } else { - getDatasourceDirectory(value); - } - }, []); - const handleRefresh = useCallback(async () => { - setState({ loading: true }); - if (!activeId) { - getLocalFiles(); - return; - } - const _path = !path || path === '/' ? '' : path; - getDatasourceDirectory(activeId, _path); - }, [activeId, path]); - const handleConfirm = useCallback(async () => { - if(activeItem && activeId === IDatasourceType.local) { - // select local file - onConfirm(activeItem, state); - } else { - setState({ loading: true }); - const _path = `${path === '/' ? '' : path}${activeItem.name}`; - const data = await previewFile({ id: activeId, path: _path }); - const item = { - name: activeItem.name, - withHeader: false, - delimiter: ',', - sample: data.contents.join('\r\n'), - path: _path, - } as any; - if(activeId !== IDatasourceType.local) { - const { sftpConfig, s3Config } = options.find((item) => item.id === activeId); - item.sftpConfig = sftpConfig; - item.s3Config = s3Config; - } - setState({ loading: false }); - onConfirm(item, state); - } - }, [activeItem, activeId, path]); - return -
- {intl.get('import.datasourceType')} - -
-
- {intl.get('import.filePath')} -
-
{path}
-
-
-
-
-
- {directory.map((item) => ( -
setState({ activeItem: item })} - onDoubleClick={() => handleSelectFile(item)} - > - {item.type === 'directory' ? : } -
- {item.name} - {item.type === 'directory' ? intl.get('import.directory') : getFileSize(item.size)} -
-
- ))} -
-
- -
- ; -}); const FileSelectModal = (props: IModalProps) => { const { visible, onConfirm, onCancel, cachedDatasourceState } = props; const { intl } = useI18n(); diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx index 4b15d9a3..07f483e1 100644 --- a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx +++ b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/index.tsx @@ -46,6 +46,7 @@ const VIDSetting = observer((props: { data.update({ [idKey]: index })} file={data.file} + data={data[idKey]} multipleMode={true} > {(!data[idKey] || data[idKey].length === 0) ? intl.get('import.selectCsvColumn') : `Column ${data[idKey]}`} @@ -161,6 +162,7 @@ const FileMapping = (props: IProps) => { updateFilePropMapping(propIndex, columnIndex as number)} file={file} + data={item[propIndex]?.mapping} > {!mappingIndex && mappingIndex !== 0 ? intl.get('import.choose') : `Column ${mappingIndex}`} diff --git a/app/stores/import.ts b/app/stores/import.ts index 5e4dd44a..91373c5b 100644 --- a/app/stores/import.ts +++ b/app/stores/import.ts @@ -45,8 +45,8 @@ export class TagFileItem { export class EdgeFileItem { file: IImportFile; props = observable.array([]); - srcIdIndex = observable.array([]); - dstIdIndex = observable.array([]); + srcIdIndex = observable.array([]); + dstIdIndex = observable.array([]); srcIdFunction?: string; dstIdFunction?: string; srcIdPrefix?: string; diff --git a/app/utils/import.ts b/app/utils/import.ts index 1a4f292c..8f3e8d77 100644 --- a/app/utils/import.ts +++ b/app/utils/import.ts @@ -57,11 +57,11 @@ export function configToJson(payload: IConfig) { } const getIdConfig = (payload: { - indexes, - prefix, - suffix, - vidFunction, - type + indexes: number[], + prefix?: string, + suffix?: string, + vidFunction?: string, + type: string }) => { const { indexes, prefix, suffix, vidFunction, type } = payload; const id = { @@ -127,26 +127,9 @@ export function edgeDataToJSON( }, edges, } as any; - const { sftpConfig, s3Config } = file; - if(s3Config) { - const { accessKey, accessSecret, bucket, endpoint, region } = s3Config; - edgeConfig.s3 = { - accessKey, - secretKey: accessSecret, - endpoint, - bucket, - region, - key: file.path - }; - } else if(sftpConfig) { - const { host, port, username, password } = sftpConfig; - edgeConfig.sftp = { - host, - port, - username, - password, - path: file.path - }; + if(file.datasourceId) { + edgeConfig.datasourceId = file.datasourceId; + edgeConfig.datasourceFilePath = file.path; } else { edgeConfig.path = file.name; } @@ -195,26 +178,9 @@ export function tagDataToJSON( }, tags } as any; - const { sftpConfig, s3Config } = file; - if(s3Config) { - const { accessKey, accessSecret, bucket, endpoint, region } = s3Config; - result.s3 = { - accessKey, - secretKey: accessSecret, - endpoint, - bucket, - region, - key: file.path - }; - } else if(sftpConfig) { - const { host, port, username, password } = sftpConfig; - result.sftp = { - host, - port, - username, - password, - path: file.path - }; + if(file.datasourceId) { + result.datasourceId = file.datasourceId; + result.datasourceFilePath = file.path; } else { result.path = file.name; } diff --git a/server/api/studio/internal/service/datasource.go b/server/api/studio/internal/service/datasource.go index 13c48097..f4f3dd0d 100644 --- a/server/api/studio/internal/service/datasource.go +++ b/server/api/studio/internal/service/datasource.go @@ -4,9 +4,7 @@ import ( "context" "encoding/json" "fmt" - "net/url" "strconv" - "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -60,6 +58,14 @@ func (d *datasourceService) Add(request types.DatasourceAddRequest) (*types.Data switch request.Type { case "s3": c := request.S3Config + endpoint, parsedBucket, err := utils.ParseEndpoint(c.Endpoint) + if err != nil { + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err) + } + if parsedBucket != "" && c.Bucket != parsedBucket { + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err, "The bucket name does not match the bucket in the endpoint") + } + c.Endpoint = endpoint if err := d.testConnectionS3(c.Endpoint, c.Region, c.Bucket, c.AccessKey, c.AccessSecret); err != nil { return nil, err } @@ -113,6 +119,14 @@ func (d *datasourceService) Update(request types.DatasourceUpdateRequest) error if c.AccessSecret == "" { c.AccessSecret = dbs.Secret } + endpoint, parsedBucket, err := utils.ParseEndpoint(c.Endpoint) + if err != nil { + return ecode.WithErrorMessage(ecode.ErrBadRequest, err) + } + if parsedBucket != "" && c.Bucket != parsedBucket { + return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "The bucket name does not match the bucket in the endpoint") + } + c.Endpoint = endpoint if err := d.testConnectionS3(c.Endpoint, c.Region, c.Bucket, c.AccessKey, c.AccessSecret); err != nil { return err } @@ -345,52 +359,7 @@ func (d *datasourceService) getFileStore(dbs *db.Datasource) (filestore.FileStor return store, nil } -func parseEndpoint(rawEndpoint string) (string, string, error) { - // endpointURL := "https://s3..amazonaws.com" - // endpointURL := "https://my-bucket.s3..amazonaws.com" - // endpointURL := "https://s3..amazonaws.com/my-bucket" - if !strings.HasPrefix(rawEndpoint, "https://") && !strings.HasPrefix(rawEndpoint, "http://") { - rawEndpoint = fmt.Sprintf("https://%s", rawEndpoint) - } - u, err := url.Parse(rawEndpoint) - if err != nil { - return "", "", err - } - host := u.Hostname() - parts := strings.SplitN(host, ".", 2) - var ( - bucket string - endpoint string - ) - if parts[0] == "s3" { - // Format: https://s3..amazonaws.com - endpoint = u.Host - if u.Path != "" { - pathParts := strings.SplitN(u.Path, "/", 3) - bucket = pathParts[1] - } - } else { - // Format: https://.s3..amazonaws.com or https://s3.amazonaws.com/ - if parts[0] == "s3.amazonaws" { - // Format: https://s3.amazonaws.com/ - endpoint = fmt.Sprintf("https://%s", u.Host) - if u.Path != "" { - pathParts := strings.SplitN(u.Path, "/", 3) - bucket = pathParts[1] - } - } else { - // Format: https://.s3..amazonaws.com - bucket = parts[0] - endpoint = fmt.Sprintf("https://%s", parts[1]) - } - } - return endpoint, bucket, nil -} func (d *datasourceService) testConnectionS3(endpoint, region, bucket, key, secret string) error { - endpoint, parsedBucket, err := parseEndpoint(endpoint) - if err != nil { - return ecode.WithErrorMessage(ecode.ErrBadRequest, err) - } sess, err := session.NewSession(&aws.Config{ Region: aws.String(region), Endpoint: aws.String(endpoint), @@ -401,9 +370,6 @@ func (d *datasourceService) testConnectionS3(endpoint, region, bucket, key, secr } svc := s3.New(sess) - if bucket != parsedBucket { - return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "The bucket name does not match the bucket in the endpoint") - } _, err = svc.HeadBucket(&s3.HeadBucketInput{ Bucket: aws.String(bucket), }) diff --git a/server/api/studio/internal/service/import.go b/server/api/studio/internal/service/import.go index 36a2545a..f944317c 100644 --- a/server/api/studio/internal/service/import.go +++ b/server/api/studio/internal/service/import.go @@ -13,6 +13,9 @@ import ( "strconv" "sync" + db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/utils" + "github.com/vesoft-inc/go-pkg/middleware" "github.com/vesoft-inc/nebula-importer/v4/pkg/config" configv3 "github.com/vesoft-inc/nebula-importer/v4/pkg/config/v3" @@ -50,8 +53,9 @@ type ( importService struct { logx.Logger - ctx context.Context - svcCtx *svc.ServiceContext + ctx context.Context + svcCtx *svc.ServiceContext + gormErrorWrapper utils.GormErrorWrapper } ) @@ -62,6 +66,57 @@ func NewImportService(ctx context.Context, svcCtx *svc.ServiceContext) ImportSer svcCtx: svcCtx, } } + +func (i *importService) updateDatasourceConfig(conf *types.CreateImportTaskRequest) (*types.ImportTaskConfig, error) { + config := conf.Config + for _, source := range config.Sources { + if source.DatasourceId != nil { + var dbs db.Datasource + result := db.CtxDB.Where("id = ?", source.DatasourceId).First(&dbs) + if result.Error != nil { + return nil, i.gormErrorWrapper(result.Error) + } + if result.RowsAffected == 0 { + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, nil, "datasource don't exist") + } + + secret, err := utils.Decrypt(dbs.Secret, []byte(cipher)) + if err != nil { + return nil, err + } + switch dbs.Type { + case "s3": + s3Config := &types.DatasourceS3Config{} + jsonConfig := dbs.Config + if err := json.Unmarshal([]byte(jsonConfig), s3Config); err != nil { + return nil, ecode.WithInternalServer(err, "get datasource config failed") + } + source.S3 = &types.S3Config{ + AccessKey: s3Config.AccessKey, + SecretKey: string(secret), + Bucket: s3Config.Bucket, + Region: s3Config.Region, + Key: *source.DatasourceFilePath, + } + + case "sftp": + sftpConfig := &types.DatasourceSFTPConfig{} + jsonConfig := dbs.Config + if err := json.Unmarshal([]byte(jsonConfig), sftpConfig); err != nil { + return nil, ecode.WithInternalServer(err, "get datasource config failed") + } + source.SFTP = &types.SFTPConfig{ + Host: sftpConfig.Host, + Port: sftpConfig.Port, + User: sftpConfig.Username, + Password: string(secret), + Path: *source.DatasourceFilePath, + } + } + } + } + return &config, nil +} func updateConfig(conf config.Configurator, taskDir, uploadDir string) { confv3 := conf.(*configv3.Config) if confv3.Log == nil { @@ -71,16 +126,21 @@ func updateConfig(conf config.Configurator, taskDir, uploadDir string) { confv3.Log.Files = append(confv3.Log.Files, filepath.Join(taskDir, importLogName)) for _, source := range confv3.Sources { - source.SourceConfig.Local.Path = filepath.Join(uploadDir, source.SourceConfig.Local.Path) + if source.SourceConfig.Local != nil { + source.SourceConfig.Local.Path = filepath.Join(uploadDir, source.SourceConfig.Local.Path) + } } } func (i *importService) CreateImportTask(req *types.CreateImportTaskRequest) (*types.CreateImportTaskData, error) { - jsons, err := json.Marshal(req.Config) + _config, err := i.updateDatasourceConfig(req) + if err != nil { + return nil, ecode.WithErrorMessage(ecode.ErrInternalServer, err) + } + jsons, err := json.Marshal(_config) if err != nil { return nil, ecode.WithErrorMessage(ecode.ErrParam, err) } - conf, err := config.FromBytes(jsons) if err != nil { diff --git a/server/api/studio/internal/service/importer/taskmgr.go b/server/api/studio/internal/service/importer/taskmgr.go index d3824e06..1f01ce6d 100644 --- a/server/api/studio/internal/service/importer/taskmgr.go +++ b/server/api/studio/internal/service/importer/taskmgr.go @@ -51,10 +51,21 @@ func CreateConfigFile(taskdir string, cfgBytes []byte) error { // erase user information _config := confv3 - _config.Client.User = "YOUR_NEBULA_NAME" - _config.Client.Password = "YOUR_NEBULA_PASSWORD" - _config.Client.Address = "" - // TODO hide data source access key and so on + _config.Client.User = "${YOUR_NEBULA_NAME}" + _config.Client.Password = "${YOUR_NEBULA_PASSWORD}" + _config.Client.Address = "${YOUR_NEBULA_ADDRESS}" + for _, source := range _config.Sources { + S3Config := source.SourceConfig.S3 + SFTPConfig := source.SourceConfig.SFTP + if S3Config != nil { + S3Config.AccessKey = "${YOUR_S3_ACCESS_KEY}" + S3Config.SecretKey = "${YOUR_S3_SECRET_KEY}" + } + if SFTPConfig != nil { + SFTPConfig.User = "${YOUR_SFTP_USER}" + SFTPConfig.Password = "${YOUR_SFTP_PASSWORD}" + } + } outYaml, err := yaml.Marshal(confv3) if err != nil { return ecode.WithErrorMessage(ecode.ErrInternalServer, err) diff --git a/server/api/studio/internal/types/types.go b/server/api/studio/internal/types/types.go index ab44c6b2..27b3babf 100644 --- a/server/api/studio/internal/types/types.go +++ b/server/api/studio/internal/types/types.go @@ -124,7 +124,7 @@ type LocalConfig struct { type ImportTaskConfig struct { Client Client `json:"client" validate:"required"` Manager Manager `json:"manager" validate:"required"` - Sources []Sources `json:"sources" validate:"required"` + Sources []*Source `json:"sources" validate:"required"` Log *Log `json:"log,omitempty,optional"` } @@ -147,13 +147,15 @@ type Manager struct { StatsInterval *string `json:"statsInterval,omitempty,optional"` } -type Sources struct { - CSV ImportTaskCSV `json:"csv" validate:"required"` - Path string `json:"path,optional,omitempty"` - S3 *S3Config `json:"s3,optional,omitempty"` - SFTP *SFTPConfig `json:"sftpConfig,optional,omitempty"` - Tags []Tag `json:"tags,optional"` - Edges []Edge `json:"edges,optional"` +type Source struct { + CSV ImportTaskCSV `json:"csv" validate:"required"` + Path string `json:"path,optional,omitempty"` + S3 *S3Config `json:"s3,optional,omitempty"` + SFTP *SFTPConfig `json:"sftp,optional,omitempty"` + DatasourceId *int64 `json:"datasourceId,optional,omitempty"` + DatasourceFilePath *string `json:"datasourceFilePath,optional,omitempty"` + Tags []Tag `json:"tags,optional"` + Edges []Edge `json:"edges,optional"` } type Log struct { diff --git a/server/api/studio/pkg/utils/datasourceParse.go b/server/api/studio/pkg/utils/datasourceParse.go new file mode 100644 index 00000000..7948babc --- /dev/null +++ b/server/api/studio/pkg/utils/datasourceParse.go @@ -0,0 +1,49 @@ +package utils + +import ( + "fmt" + "net/url" + "strings" +) + +func ParseEndpoint(rawEndpoint string) (string, string, error) { + // endpointURL := "https://s3..amazonaws.com" + // endpointURL := "https://my-bucket.s3..amazonaws.com" + // endpointURL := "https://s3..amazonaws.com/my-bucket" + if !strings.HasPrefix(rawEndpoint, "https://") && !strings.HasPrefix(rawEndpoint, "http://") { + rawEndpoint = fmt.Sprintf("https://%s", rawEndpoint) + } + u, err := url.Parse(rawEndpoint) + if err != nil { + return "", "", err + } + host := u.Hostname() + parts := strings.SplitN(host, ".", 2) + var ( + bucket string + endpoint string + ) + if parts[0] == "s3" { + // Format: https://s3..amazonaws.com + endpoint = fmt.Sprintf("https://%s", u.Host) + if u.Path != "" { + pathParts := strings.SplitN(u.Path, "/", 3) + bucket = pathParts[1] + } + } else { + // Format: https://.s3..amazonaws.com or https://s3.amazonaws.com/ + if parts[0] == "s3.amazonaws" { + // Format: https://s3.amazonaws.com/ + endpoint = fmt.Sprintf("https://%s", u.Host) + if u.Path != "" { + pathParts := strings.SplitN(u.Path, "/", 3) + bucket = pathParts[1] + } + } else { + // Format: https://.s3..amazonaws.com + bucket = parts[0] + endpoint = fmt.Sprintf("https://%s", parts[1]) + } + } + return endpoint, bucket, nil +} diff --git a/server/api/studio/restapi/import.api b/server/api/studio/restapi/import.api index 3480ab0a..0aaeb894 100644 --- a/server/api/studio/restapi/import.api +++ b/server/api/studio/restapi/import.api @@ -76,7 +76,7 @@ type ( ImportTaskConfig { Client Client `json:"client" validate:"required"` Manager Manager `json:"manager" validate:"required"` - Sources []Sources `json:"sources" validate:"required"` + Sources []*Source `json:"sources" validate:"required"` Log *Log `json:"log,omitempty,optional"` } @@ -99,13 +99,15 @@ type ( StatsInterval *string `json:"statsInterval,omitempty,optional"` } - Sources { - CSV ImportTaskCSV `json:"csv" validate:"required"` - Path string `json:"path,optional,omitempty"` - S3 *S3Config `json:"s3,optional,omitempty"` - SFTP *SFTPConfig `json:"sftpConfig,optional,omitempty"` - Tags []Tag `json:"tags,optional"` - Edges []Edge `json:"edges,optional"` + Source { + CSV ImportTaskCSV `json:"csv" validate:"required"` + Path string `json:"path,optional,omitempty"` + S3 *S3Config `json:"s3,optional,omitempty"` + SFTP *SFTPConfig `json:"sftp,optional,omitempty"` + DatasourceId *int64 `json:"datasourceId,optional,omitempty"` + DatasourceFilePath *string `json:"datasourceFilePath,optional,omitempty"` + Tags []Tag `json:"tags,optional"` + Edges []Edge `json:"edges,optional"` } Log { From 59247adfa534205a61937616e8d2f5d07308ea05 Mon Sep 17 00:00:00 2001 From: hetao92 <18328704+hetao92@users.noreply.github.com> Date: Thu, 23 Mar 2023 09:39:15 +0000 Subject: [PATCH 09/11] mod: add platform in config && modify datasource model --- app/config/locale/en-US.json | 5 +- app/config/locale/zh-CN.json | 4 +- app/interfaces/datasource.ts | 28 ++ .../PlatformConfig/S3ConfigForm.tsx | 56 ++-- .../PlatformConfig/index.module.less | 3 - .../DatasourceConfig/PlatformConfig/index.tsx | 14 +- .../DatasourceList/RemoteList/index.tsx | 10 +- app/stores/datasource.ts | 9 +- .../api/studio/internal/model/datasource.go | 1 + .../api/studio/internal/service/datasource.go | 265 ++++++++---------- server/api/studio/internal/service/import.go | 30 +- .../studio/internal/service/importer/task.go | 3 + .../internal/service/importer/taskmgr.go | 5 + server/api/studio/internal/types/types.go | 34 ++- server/api/studio/pkg/filestore/filestore.go | 8 +- server/api/studio/pkg/filestore/ossStore.go | 122 ++++++++ server/api/studio/pkg/filestore/s3store.go | 35 ++- server/api/studio/pkg/filestore/sftpstore.go | 6 +- .../api/studio/pkg/utils/datasourceParse.go | 6 +- server/api/studio/restapi/datasource.api | 25 +- server/api/studio/restapi/import.api | 9 + server/go.mod | 2 +- 22 files changed, 451 insertions(+), 229 deletions(-) create mode 100644 server/api/studio/pkg/filestore/ossStore.go diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json index 63b7d937..5d19b4d3 100644 --- a/app/config/locale/en-US.json +++ b/app/config/locale/en-US.json @@ -297,7 +297,10 @@ "datasourceType": "Data Source Type", "filePath": "File Path", "directory": "Directory", - "preview": "Preview" + "preview": "Preview", + "customize": "Customize", + "s3Platform": "S3 Platform", + "account": "Account" }, "schema": { "spaceList": "Graph Space List", diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json index 83d8f656..1913f01e 100644 --- a/app/config/locale/zh-CN.json +++ b/app/config/locale/zh-CN.json @@ -298,7 +298,9 @@ "datasourceType": "数据源类型", "filePath": "文件路径", "directory": "目录", - "preview": "预览" + "preview": "预览", + "customize": "自定义", + "s3Platform": "S3 平台" }, "schema": { "spaceList": "图空间列表", diff --git a/app/interfaces/datasource.ts b/app/interfaces/datasource.ts index 5ed4699a..fc01748b 100644 --- a/app/interfaces/datasource.ts +++ b/app/interfaces/datasource.ts @@ -3,3 +3,31 @@ export enum IDatasourceType { 'sftp' = 'sftp', 'local' = 'local' } +export enum IS3Platform { + 'aws' = 'aws', + 'oss' = 'oss', + 'customize' = 'customize', + 'tecent' = 'tecent' +} + +export interface IDatasourceAdd { + name: string; + type: IDatasourceType; + platform?: string; + s3Config?: { + accessKey: string; + accessSecret?: string; + endpoint: string; + bucket: string; + region?: string; + }; + sftpConfig?: { + host: string; + port: number; + username: string; + password: string; + }; +} +export interface IDatasourceUpdate extends IDatasourceAdd { + id: number; +} \ No newline at end of file diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx index 22ebae53..e3d04b9f 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx @@ -2,7 +2,7 @@ import { useI18n } from '@vesoft-inc/i18n'; import { Input, Form, Select, FormInstance } from 'antd'; import React, { useEffect, useState } from 'react'; import { observer } from 'mobx-react-lite'; -import styles from './index.module.less'; +import { IS3Platform } from '@app/interfaces/datasource'; const FormItem = Form.Item; interface IProps { formRef: FormInstance; @@ -16,7 +16,6 @@ const S3ConfigForm = (props: IProps) => { useEffect(() => { if(mode === 'edit') { formRef.setFieldValue(['s3Config', 'accessSecret'], tempPwd); - formRef.setFieldValue('platform', 'aws'); } }, [mode]); @@ -27,31 +26,40 @@ const S3ConfigForm = (props: IProps) => { setFlag(true); } }; + return ( - - - - - - - - - - - - - - - - + + - - + + {({ getFieldValue }) => { + const platform = getFieldValue('platform'); + return ( + <> + {platform === IS3Platform.aws && + + } + + .amazonaws.com' : intl.get('import.enterAddress')} /> + + + + + + + + + + + + ); + }} ); diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less index 68b520a8..39e7418b 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less @@ -9,9 +9,6 @@ padding: 0 10px 35px; } } - .mixedItem { - margin-bottom: 0; - } .btns { text-align: center; :global(.ant-btn) { diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx index 0bbd9294..b4620c2e 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx @@ -34,29 +34,28 @@ const DatasourceConfigModal = (props: IProps) => { const tempPwd = useMemo(() => uuidv4() + Date.now(), []); const mode = useMemo(() => (data ? 'edit' : 'create'), [data]); const submit = async (values: any) => { - const { platform, ...rest } = values; const _type = values.type || type; setLoading(true); if(mode === 'create') { const flag = await addDataSource({ type: _type, name: '', - ...rest + ...values }); setLoading(false); flag && (message.success(intl.get('schema.createSuccess')), onConfirm()); } else { const _config = _type === IDatasourceType.s3 ? 's3Config' : 'sftpConfig'; - if(_type === IDatasourceType.s3 && rest[_config].accessSecret === tempPwd) { - delete rest[_config].accessSecret; - } else if (_type === IDatasourceType.sftp && rest[_config].password === tempPwd) { - delete rest[_config].password; + if(_type === IDatasourceType.s3 && values[_config].accessSecret === tempPwd) { + delete values[_config].accessSecret; + } else if (_type === IDatasourceType.sftp && values[_config].password === tempPwd) { + delete values[_config].password; } const flag = await updateDataSource({ id: data.id, type: _type, name: '', - ...rest + ...values }); setLoading(false); flag && (message.success(intl.get('common.updateSuccess')), onConfirm()); @@ -74,7 +73,6 @@ const DatasourceConfigModal = (props: IProps) => { footer={false} >
{({ getFieldValue }) => { diff --git a/app/pages/Import/DatasourceList/RemoteList/index.tsx b/app/pages/Import/DatasourceList/RemoteList/index.tsx index c476daf5..3017199c 100644 --- a/app/pages/Import/DatasourceList/RemoteList/index.tsx +++ b/app/pages/Import/DatasourceList/RemoteList/index.tsx @@ -69,10 +69,14 @@ const DatasourceList = (props: IProps) => { const [visible, setVisible] = useState(false); const [selectItems, setSelectItems] = useState([]); - const editItem = (item) => { + const create = useCallback(() => { + setEditData(null); + setVisible(true); + }, []); + const editItem = useCallback((item) => { setEditData(item); setVisible(true); - }; + }, []); const deleteItem = useCallback(async id => { const flag = await deleteDataSource(id); flag && (getList(), message.success(intl.get('common.deleteSuccess'))); @@ -129,7 +133,7 @@ const DatasourceList = (props: IProps) => { return (
- Object.prototype.hasOwnProperty.call(this, key) && (this[key] = payload[key])); }; - addDataSource = async (payload) => { + addDataSource = async (payload: IDatasourceAdd) => { const { code } = await service.addDatasource(payload); return code === 0; }; - updateDataSource = async (payload) => { + updateDataSource = async (payload: IDatasourceUpdate) => { const { code } = await service.updateDatasource(payload); return code === 0; }; - getDatasourceList = async (payload?: { type?: string }) => { + getDatasourceList = async (payload?: { type?: IDatasourceType }) => { const { code, data } = await service.getDatasourceList(payload); if(code === 0) { return data.list; @@ -51,7 +52,7 @@ export class DatasourceStore { } }; previewFile = async (payload: { - id: string, + id: number, path?: string, }) => { const { id, path } = payload; diff --git a/server/api/studio/internal/model/datasource.go b/server/api/studio/internal/model/datasource.go index b6b95599..2c97f08d 100644 --- a/server/api/studio/internal/model/datasource.go +++ b/server/api/studio/internal/model/datasource.go @@ -6,6 +6,7 @@ type Datasource struct { ID int `json:"id" gorm:"primaryKey;autoIncrement"` Name string `json:"name"` Type string `json:"type"` + Platform string `json:"platform"` Config string `json:"config"` Secret string `json:"secret"` Host string `json:"host"` diff --git a/server/api/studio/internal/service/datasource.go b/server/api/studio/internal/service/datasource.go index f4f3dd0d..a6680f94 100644 --- a/server/api/studio/internal/service/datasource.go +++ b/server/api/studio/internal/service/datasource.go @@ -3,15 +3,10 @@ package service import ( "context" "encoding/json" + "errors" "fmt" "strconv" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/pkg/sftp" db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" @@ -20,7 +15,6 @@ import ( "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/filestore" "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/utils" "github.com/zeromicro/go-zero/core/logx" - "golang.org/x/crypto/ssh" ) type ( @@ -55,56 +49,28 @@ func NewDatasourceService(ctx context.Context, svcCtx *svc.ServiceContext) Datas } func (d *datasourceService) Add(request types.DatasourceAddRequest) (*types.DatasourceAddData, error) { - switch request.Type { + typ := request.Type + platform := request.Platform + var cfg interface{} + switch typ { case "s3": - c := request.S3Config - endpoint, parsedBucket, err := utils.ParseEndpoint(c.Endpoint) - if err != nil { - return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err) - } - if parsedBucket != "" && c.Bucket != parsedBucket { - return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err, "The bucket name does not match the bucket in the endpoint") - } - c.Endpoint = endpoint - if err := d.testConnectionS3(c.Endpoint, c.Region, c.Bucket, c.AccessKey, c.AccessSecret); err != nil { - return nil, err - } - secret := c.AccessSecret - c.AccessSecret = "" - cstr, err := json.Marshal(c) - if err != nil { - return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err, "json stringify error") - } - crypto, err := utils.Encrypt([]byte(secret), []byte(cipher)) - id, err := d.save(request.Type, request.Name, string(cstr), crypto) - if err != nil { - return nil, err - } - return &types.DatasourceAddData{ - ID: id, - }, nil + cfg = request.S3Config case "sftp": - c := request.SFTPConfig - if err := d.testConnectionSFTP(c.Host, c.Port, c.Username, c.Password); err != nil { - return nil, err - } - pwd := c.Password - c.Password = "" - cstr, err := json.Marshal(c) - if err != nil { - d.Logger.Errorf("json stringify error", c) - return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err, "json stringify error") - } - crypto, err := utils.Encrypt([]byte(pwd), []byte(cipher)) - id, err := d.save(request.Type, request.Name, string(cstr), crypto) - if err != nil { - return nil, err - } - return &types.DatasourceAddData{ - ID: id, - }, nil + cfg = request.SFTPConfig + default: + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, nil, "Invalid datasource type") + } + cfgStr, crypto, err := validate(typ, platform, cfg) + if err != nil { + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err) + } + id, err := d.save(request.Type, request.Name, request.Platform, cfgStr, crypto) + if err != nil { + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err) } - return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, nil, "datasource type can't support'") + return &types.DatasourceAddData{ + ID: id, + }, nil } func (d *datasourceService) Update(request types.DatasourceUpdateRequest) error { @@ -113,58 +79,45 @@ func (d *datasourceService) Update(request types.DatasourceUpdateRequest) error if err != nil { return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "find data error") } - switch request.Type { + typ := request.Type + platform := request.Platform + var cfg interface{} + switch typ { case "s3": - c := request.S3Config - if c.AccessSecret == "" { - c.AccessSecret = dbs.Secret - } - endpoint, parsedBucket, err := utils.ParseEndpoint(c.Endpoint) - if err != nil { - return ecode.WithErrorMessage(ecode.ErrBadRequest, err) - } - if parsedBucket != "" && c.Bucket != parsedBucket { - return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "The bucket name does not match the bucket in the endpoint") + s3Config := request.S3Config + if s3Config.AccessSecret == "" { + s3Config.AccessSecret = dbs.Secret } - c.Endpoint = endpoint - if err := d.testConnectionS3(c.Endpoint, c.Region, c.Bucket, c.AccessKey, c.AccessSecret); err != nil { - return err + cfg = &types.DatasourceS3Config{ + AccessKey: s3Config.AccessKey, + AccessSecret: s3Config.AccessSecret, + Endpoint: s3Config.Endpoint, + Bucket: s3Config.Bucket, + Region: s3Config.Region, } - secret := c.AccessSecret - c.AccessSecret = "" - cstr, err := json.Marshal(c) - if err != nil { - return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "json stringify error") - } - crypto, err := utils.Encrypt([]byte(secret), []byte(cipher)) - err = d.update(datasourceId, request.Type, request.Name, string(cstr), crypto) - if err != nil { - return err - } - return nil case "sftp": - c := request.SFTPConfig - if c.Password == "" { - c.Password = dbs.Secret - } - if err := d.testConnectionSFTP(c.Host, c.Port, c.Username, c.Password); err != nil { - return err - } - pwd := c.Password - c.Password = "" - cstr, err := json.Marshal(c) - if err != nil { - d.Logger.Errorf("json stringify error", c) - return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "json stringify error") + sftpCfg := request.SFTPConfig + if sftpCfg.Password == "" { + sftpCfg.Password = dbs.Secret } - crypto, err := utils.Encrypt([]byte(pwd), []byte(cipher)) - err = d.update(datasourceId, request.Type, request.Name, string(cstr), crypto) - if err != nil { - return err + cfg = &types.DatasourceSFTPConfig{ + Host: sftpCfg.Host, + Port: sftpCfg.Port, + Username: sftpCfg.Username, + Password: sftpCfg.Password, } - return nil + default: + return ecode.WithErrorMessage(ecode.ErrBadRequest, nil, "Invalid datasource type") + } + cfgStr, crypto, err := validate(typ, platform, cfg) + if err != nil { + return err } - return ecode.WithErrorMessage(ecode.ErrBadRequest, nil, "datasource type can't support'") + err = d.update(datasourceId, request.Type, request.Platform, request.Name, cfgStr, crypto) + if err != nil { + return err + } + return nil } func (d *datasourceService) List(request types.DatasourceListRequest) (*types.DatasourceData, error) { @@ -185,6 +138,7 @@ func (d *datasourceService) List(request types.DatasourceListRequest) (*types.Da config := types.DatasourceConfig{ ID: item.ID, Type: item.Type, + Platform: item.Platform, Name: item.Name, CreateTime: item.CreateTime.UnixMilli(), } @@ -310,11 +264,12 @@ func (d *datasourceService) findOne(datasourceId int) (*db.Datasource, error) { return &dbs, nil } -func (d *datasourceService) save(typ, name, config, secret string) (id int, err error) { +func (d *datasourceService) save(typ, name, platform, config, secret string) (id int, err error) { user := d.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) host := user.Address + ":" + strconv.Itoa(user.Port) dbs := &db.Datasource{ Type: typ, + Platform: platform, Name: name, Config: config, Secret: secret, @@ -327,12 +282,13 @@ func (d *datasourceService) save(typ, name, config, secret string) (id int, err } return int(dbs.ID), nil } -func (d *datasourceService) update(id int, typ, name, config, secret string) (err error) { +func (d *datasourceService) update(id int, typ, platform, name, config, secret string) (err error) { user := d.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) host := user.Address + ":" + strconv.Itoa(user.Port) result := db.CtxDB.Model(&db.Datasource{ID: id}).Updates(map[string]interface{}{ "type": typ, "name": name, + "platform": platform, "config": config, "secret": secret, "host": host, @@ -350,7 +306,7 @@ func (d *datasourceService) getFileStore(dbs *db.Datasource) (filestore.FileStor if err := json.Unmarshal([]byte(dbs.Config), &config); err != nil { return nil, ecode.WithInternalServer(err, "parse the datasource config error") } - store, err := filestore.NewFileStore(dbs.Type, dbs.Config, dbs.Secret) + store, err := filestore.NewFileStore(dbs.Type, dbs.Config, dbs.Secret, dbs.Platform) if err != nil { d.Logger.Errorf("create the file store error") return nil, ecode.WithInternalServer(err, "create the file store error") @@ -359,61 +315,86 @@ func (d *datasourceService) getFileStore(dbs *db.Datasource) (filestore.FileStor return store, nil } -func (d *datasourceService) testConnectionS3(endpoint, region, bucket, key, secret string) error { - sess, err := session.NewSession(&aws.Config{ - Region: aws.String(region), - Endpoint: aws.String(endpoint), - Credentials: credentials.NewStaticCredentials(key, secret, ""), - }) +func formatDatasourceConfig(config interface{}, password string) (string, string, error) { + cfgStr, err := json.Marshal(config) if err != nil { - return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "failed to create session") + return "", "", fmt.Errorf("json stringify config error: %v", err) } - - svc := s3.New(sess) - _, err = svc.HeadBucket(&s3.HeadBucketInput{ - Bucket: aws.String(bucket), - }) + crypto, err := utils.Encrypt([]byte(password), []byte(cipher)) if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - if awsErr.Code() == "NotFound" { - return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "Bucket does not exist") - } - } - return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "Failed to head bucket") + return "", "", fmt.Errorf("encrypt password error: %v", err) } - - return nil + return string(cfgStr), crypto, nil } -func (d *datasourceService) testConnectionSFTP(host string, port int, username string, password string) error { - // create an SSH client config - config := &ssh.ClientConfig{ - User: username, - Auth: []ssh.AuthMethod{ - ssh.Password(password), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), +func validate(typ string, platform string, config interface{}) (string, string, error) { + switch typ { + case "s3": + if platform == "oss" { + cfg := config.(*types.DatasourceS3Config) + err := validateOss(cfg) + if err != nil { + return "", "", err + } + secret := cfg.AccessSecret + cfg.AccessSecret = "" + cfgStr, crypto, err := formatDatasourceConfig(config, secret) + return cfgStr, crypto, err + } else { + cfg := config.(*types.DatasourceS3Config) + endpoint, parsedBucket, err := utils.ParseEndpoint(platform, cfg.Endpoint) + if err != nil { + return "", "", err + } + if parsedBucket != "" && cfg.Bucket != parsedBucket { + return "", "", errors.New("bucket name in endpoint and bucket name in config are different") + } + cfg.Endpoint = endpoint + err = validateS3(platform, cfg) + if err != nil { + return "", "", err + } + secret := cfg.AccessSecret + cfg.AccessSecret = "" + cfgStr, crypto, err := formatDatasourceConfig(config, secret) + return cfgStr, crypto, err + } + case "sftp": + cfg := config.(*types.DatasourceSFTPConfig) + err := validateSftp(cfg) + if err != nil { + return "", "", err + } + secret := cfg.Password + cfg.Password = "" + cfgStr, crypto, err := formatDatasourceConfig(config, secret) + return cfgStr, crypto, err + default: + return "", "", errors.New("unsupported datasource type") } +} - // connect to the remote SSH server - conn, err := ssh.Dial("tcp", fmt.Sprintf("%s:%d", host, port), config) +func validateOss(cfg *types.DatasourceS3Config) error { + _, err := filestore.NewOssStore(cfg.Endpoint, cfg.Bucket, cfg.AccessKey, cfg.AccessSecret) if err != nil { - return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "failed to dial SSH server") + return fmt.Errorf("connect the oss client error: %s", err) } - defer conn.Close() + return nil +} - // create an SFTP client session - client, err := sftp.NewClient(conn) +func validateSftp(cfg *types.DatasourceSFTPConfig) error { + store, err := filestore.NewSftpStore(cfg.Host, cfg.Port, cfg.Username, cfg.Password) if err != nil { - return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "failed to create SFTP session") + return fmt.Errorf("connect the sftp client error: %s", err) } - defer client.Close() + store.SftpClient.Close() + return nil +} - // test the SFTP connection by listing the remote directory - _, err = client.ReadDir("/") +func validateS3(platform string, cfg *types.DatasourceS3Config) error { + _, err := filestore.NewS3Store(platform, cfg.Endpoint, cfg.Region, cfg.Bucket, cfg.AccessKey, cfg.AccessSecret) if err != nil { - return ecode.WithErrorMessage(ecode.ErrBadRequest, err, "failed to list remote directory") + return fmt.Errorf("connect the s3 client error: %s", err) } - return nil } diff --git a/server/api/studio/internal/service/import.go b/server/api/studio/internal/service/import.go index f944317c..cfdae916 100644 --- a/server/api/studio/internal/service/import.go +++ b/server/api/studio/internal/service/import.go @@ -86,19 +86,31 @@ func (i *importService) updateDatasourceConfig(conf *types.CreateImportTaskReque } switch dbs.Type { case "s3": - s3Config := &types.DatasourceS3Config{} + cfg := &types.DatasourceS3Config{} jsonConfig := dbs.Config - if err := json.Unmarshal([]byte(jsonConfig), s3Config); err != nil { + if err := json.Unmarshal([]byte(jsonConfig), cfg); err != nil { return nil, ecode.WithInternalServer(err, "get datasource config failed") } - source.S3 = &types.S3Config{ - AccessKey: s3Config.AccessKey, - SecretKey: string(secret), - Bucket: s3Config.Bucket, - Region: s3Config.Region, - Key: *source.DatasourceFilePath, + if dbs.Platform == "oss" { + source.OSS = &types.OSSConfig{ + AccessKey: cfg.AccessKey, + SecretKey: string(secret), + Bucket: cfg.Bucket, + Endpoint: cfg.Endpoint, + Key: *source.DatasourceFilePath, + } + } else { + source.S3 = &types.S3Config{ + AccessKey: cfg.AccessKey, + SecretKey: string(secret), + Bucket: cfg.Bucket, + Region: cfg.Region, + Key: *source.DatasourceFilePath, + } + if dbs.Platform == "customize" && cfg.Region == "" { + source.S3.Region = "us-east-1" + } } - case "sftp": sftpConfig := &types.DatasourceSFTPConfig{} jsonConfig := dbs.Config diff --git a/server/api/studio/internal/service/importer/task.go b/server/api/studio/internal/service/importer/task.go index 72363abe..6befc0d0 100644 --- a/server/api/studio/internal/service/importer/task.go +++ b/server/api/studio/internal/service/importer/task.go @@ -19,6 +19,9 @@ type Task struct { } func (t *Task) UpdateQueryStats() error { + if (t.Client == nil) || (t.Client.Manager == nil) { + return nil + } stats := t.Client.Manager.Stats() t.TaskInfo.Stats = *stats return nil diff --git a/server/api/studio/internal/service/importer/taskmgr.go b/server/api/studio/internal/service/importer/taskmgr.go index 1f01ce6d..a968ed8a 100644 --- a/server/api/studio/internal/service/importer/taskmgr.go +++ b/server/api/studio/internal/service/importer/taskmgr.go @@ -57,6 +57,7 @@ func CreateConfigFile(taskdir string, cfgBytes []byte) error { for _, source := range _config.Sources { S3Config := source.SourceConfig.S3 SFTPConfig := source.SourceConfig.SFTP + OSSConfig := source.SourceConfig.OSS if S3Config != nil { S3Config.AccessKey = "${YOUR_S3_ACCESS_KEY}" S3Config.SecretKey = "${YOUR_S3_SECRET_KEY}" @@ -65,6 +66,10 @@ func CreateConfigFile(taskdir string, cfgBytes []byte) error { SFTPConfig.User = "${YOUR_SFTP_USER}" SFTPConfig.Password = "${YOUR_SFTP_PASSWORD}" } + if OSSConfig != nil { + OSSConfig.AccessKey = "${YOUR_OSS_ACCESS_KEY}" + OSSConfig.SecretKey = "${YOUR_OSS_SECRET_KEY}" + } } outYaml, err := yaml.Marshal(confv3) if err != nil { diff --git a/server/api/studio/internal/types/types.go b/server/api/studio/internal/types/types.go index 27b3babf..0e127a2f 100644 --- a/server/api/studio/internal/types/types.go +++ b/server/api/studio/internal/types/types.go @@ -117,6 +117,14 @@ type SFTPConfig struct { Path string `json:"path,omitempty"` } +type OSSConfig struct { + Endpoint string `json:"endpoint,omitempty"` + AccessKey string `json:"accessKey,omitempty"` + SecretKey string `json:"secretKey,omitempty"` + Bucket string `json:"bucket,omitempty"` + Key string `json:"key,omitempty"` +} + type LocalConfig struct { Path string `json:"path,omitempty"` } @@ -152,6 +160,7 @@ type Source struct { Path string `json:"path,optional,omitempty"` S3 *S3Config `json:"s3,optional,omitempty"` SFTP *SFTPConfig `json:"sftp,optional,omitempty"` + OSS *OSSConfig `json:"oss,optional,omitempty"` DatasourceId *int64 `json:"datasourceId,optional,omitempty"` DatasourceFilePath *string `json:"datasourceFilePath,optional,omitempty"` Tags []Tag `json:"tags,optional"` @@ -345,10 +354,10 @@ type FavoriteIDResult struct { type DatasourceS3Config struct { Endpoint string `json:"endpoint"` - Region string `json:"region"` + Region string `json:"region,optional"` Bucket string `json:"bucket"` AccessKey string `json:"accessKey"` - AccessSecret string `json:"accessSecret"` + AccessSecret string `json:"accessSecret,optional"` } type DatasourceSFTPConfig struct { @@ -359,22 +368,23 @@ type DatasourceSFTPConfig struct { } type DatasourceS3UpdateConfig struct { - Endpoint string `json:"endpoint, optional, omitempty"` - Region string `json:"region, optional, omitempty"` - Bucket string `json:"bucket, optional, omitempty"` - AccessKey string `json:"accessKey, optional, omitempty"` - AccessSecret string `json:"accessSecret, optional, omitempty"` + Endpoint string `json:"endpoint,optional,omitempty"` + Region string `json:"region,optional,omitempty"` + Bucket string `json:"bucket,optional,omitempty"` + AccessKey string `json:"accessKey,optional,omitempty"` + AccessSecret string `json:"accessSecret,optional,omitempty"` } type DatasourceSFTPUpdateConfig struct { - Host string `json:"host, optional, omitempty"` - Port int `json:"port, optional, omitempty"` - Username string `json:"username, optional, omitempty"` - Password string `json:"password, optional, omitempty"` + Host string `json:"host,optional,omitempty"` + Port int `json:"port,optional,omitempty"` + Username string `json:"username,optional,omitempty"` + Password string `json:"password,optional,omitempty"` } type DatasourceAddRequest struct { Type string `json:"type"` + Platform string `json:"platform,optional,omitempty"` Name string `json:"name"` S3Config *DatasourceS3Config `json:"s3Config,optional"` SFTPConfig *DatasourceSFTPConfig `json:"sftpConfig,optional"` @@ -382,6 +392,7 @@ type DatasourceAddRequest struct { type DatasourceUpdateRequest struct { ID int `path:"id"` + Platform string `json:"platform,optional,omitempty"` Type string `json:"type"` Name string `json:"name"` S3Config *DatasourceS3UpdateConfig `json:"s3Config,optional"` @@ -408,6 +419,7 @@ type DatasourceConfig struct { ID int `json:"id"` Type string `json:"type"` Name string `json:"name"` + Platform string `json:"platform"` S3Config *DatasourceS3Config `json:"s3Config,optional"` SFTPConfig *DatasourceSFTPConfig `json:"sftpConfig,optional"` CreateTime int64 `json:"createTime,optional"` diff --git a/server/api/studio/pkg/filestore/filestore.go b/server/api/studio/pkg/filestore/filestore.go index 5d856aa0..808c6e04 100644 --- a/server/api/studio/pkg/filestore/filestore.go +++ b/server/api/studio/pkg/filestore/filestore.go @@ -33,14 +33,18 @@ type ( } ) -func NewFileStore(typ, config, secret string) (FileStore, error) { +func NewFileStore(typ, config, secret, platform string) (FileStore, error) { switch typ { case "s3": var c S3Config if err := json.Unmarshal([]byte(config), &c); err != nil { return nil, errors.New("parse the s3 config error") } - return NewS3Store(c.Endpoint, c.Region, c.Bucket, c.AccessKey, secret) + if platform == "oss" { + return NewOssStore(c.Endpoint, c.Bucket, c.AccessKey, secret) + } else { + return NewS3Store(platform, c.Endpoint, c.Region, c.Bucket, c.AccessKey, secret) + } case "sftp": var c SftpConfig if err := json.Unmarshal([]byte(config), &c); err != nil { diff --git a/server/api/studio/pkg/filestore/ossStore.go b/server/api/studio/pkg/filestore/ossStore.go new file mode 100644 index 00000000..d250eead --- /dev/null +++ b/server/api/studio/pkg/filestore/ossStore.go @@ -0,0 +1,122 @@ +package filestore + +import ( + "bufio" + "errors" + "fmt" + "strings" + + "github.com/aliyun/aliyun-oss-go-sdk/oss" +) + +type OssStore struct { + Client *oss.Client + Bucket *oss.Bucket +} + +func NewOssStore(endpoint, bucketName, accessKey, accessSecret string) (*OssStore, error) { + client, err := oss.New(endpoint, accessKey, accessSecret) + if err != nil { + return nil, fmt.Errorf("failed to create oss client: %v", err) + } + bucket, err := client.Bucket(bucketName) + if err != nil { + return nil, fmt.Errorf("failed to get bucket: %v", err) + } + return &OssStore{ + Client: client, + Bucket: bucket, + }, nil +} + +func (s *OssStore) ReadFile(path string, startLine ...int) ([]string, error) { + var numLines int + var start int + if len(startLine) == 0 { + start = 0 + numLines = -1 + } else if len(startLine) == 1 { + start = startLine[0] + numLines = -1 + } else { + start = startLine[0] + numLines = startLine[1] + } + + resp, err := s.Bucket.GetObject(path) + if err != nil { + return nil, err + } + defer resp.Close() + + fileScanner := bufio.NewScanner(resp) + + var lines []string + for i := 0; i < start; i++ { + if !fileScanner.Scan() { + return nil, errors.New("start line is beyond end of file") + } + } + + for i := 0; numLines < 0 || i < numLines; i++ { + if !fileScanner.Scan() { + break + } + lines = append(lines, fileScanner.Text()) + } + + if err := fileScanner.Err(); err != nil { + return nil, err + } + + return lines, nil +} + +func (s *OssStore) ListFiles(path string) ([]FileConfig, error) { + resp, err := s.Bucket.ListObjectsV2(oss.Prefix(path), oss.Delimiter("/")) + if err != nil { + return nil, err + } + var files []FileConfig + + for _, str := range resp.CommonPrefixes { + name := str[:len(str)-1] // remove trailing slash + files = append(files, FileConfig{ + Name: strings.TrimPrefix(name, path), + Type: "directory", + }) + } + for _, obj := range resp.Objects { + var objType string + key := obj.Key + if key[len(key)-1:] == "/" { + objType = "directory" + } else if strings.HasSuffix(key, ".csv") { + objType = "csv" + } + name := strings.TrimPrefix(key, path) + if objType != "" && name != "" { + s3Object := FileConfig{ + Name: name, + Type: objType, + Size: obj.Size, + } + files = append(files, s3Object) + } + } + + return files, nil +} + +func (s *OssStore) ListBuckets() ([]string, error) { + resp, err := s.Client.ListBuckets() + if err != nil { + return nil, err + } + var buckets []string + for _, obj := range resp.Buckets { + buckets = append(buckets, obj.Name) + } + + return buckets, nil +} diff --git a/server/api/studio/pkg/filestore/s3store.go b/server/api/studio/pkg/filestore/s3store.go index 6145abfa..3dbf4da4 100644 --- a/server/api/studio/pkg/filestore/s3store.go +++ b/server/api/studio/pkg/filestore/s3store.go @@ -3,8 +3,10 @@ package filestore import ( "bufio" "errors" + "fmt" "strings" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -18,16 +20,39 @@ type S3Store struct { Bucket string } -func NewS3Store(endpoint, region, bucket, accessKey, accessSecret string) (*S3Store, error) { - sess, err := session.NewSession(&aws.Config{ - Region: aws.String(region), - Credentials: credentials.NewStaticCredentials(accessKey, accessSecret, ""), - }) +func NewS3Store(platform, endpoint, region, bucket, accessKey, accessSecret string) (*S3Store, error) { + var sess *session.Session + var err error + + if platform == "aws" { + sess, err = session.NewSession(&aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewStaticCredentials(accessKey, accessSecret, ""), + }) + } else { + sess, err = session.NewSession(&aws.Config{ + Region: aws.String("us-east-1"), + Credentials: credentials.NewStaticCredentials(accessKey, accessSecret, ""), + S3ForcePathStyle: aws.Bool(true), + Endpoint: aws.String(endpoint), + }) + } if err != nil { return nil, errors.New("failed to create session") } svc := s3.New(sess) + _, err = svc.HeadBucket(&s3.HeadBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "NotFound" { + return nil, fmt.Errorf("bucket does not exist: %v", err) + } + } + return nil, fmt.Errorf("failed to head bucket: %v", err) + } return &S3Store{ S3Client: svc, Bucket: bucket, diff --git a/server/api/studio/pkg/filestore/sftpstore.go b/server/api/studio/pkg/filestore/sftpstore.go index 93e9db47..1074170e 100644 --- a/server/api/studio/pkg/filestore/sftpstore.go +++ b/server/api/studio/pkg/filestore/sftpstore.go @@ -31,12 +31,12 @@ func NewSftpStore(host string, port int, username string, password string) (*Sft addr := fmt.Sprintf("%s:%d", host, port) conn, err := ssh.Dial("tcp", addr, config) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to dial SSH server: %s", err) } client, err := sftp.NewClient(conn) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create SFTP client: %s", err) } return &SftpStore{ @@ -109,7 +109,7 @@ func (s *SftpStore) ListFiles(dir string) ([]FileConfig, error) { isDir := file.IsDir() name := file.Name() var fileType string - if isDir { + if isDir && !strings.HasPrefix(name, ".") { fileType = "directory" } else if strings.HasSuffix(name, ".csv") { fileType = "csv" diff --git a/server/api/studio/pkg/utils/datasourceParse.go b/server/api/studio/pkg/utils/datasourceParse.go index 7948babc..b7a9d5d9 100644 --- a/server/api/studio/pkg/utils/datasourceParse.go +++ b/server/api/studio/pkg/utils/datasourceParse.go @@ -6,7 +6,10 @@ import ( "strings" ) -func ParseEndpoint(rawEndpoint string) (string, string, error) { +func ParseEndpoint(platform, rawEndpoint string) (string, string, error) { + if platform != "aws" { + return rawEndpoint, "", nil + } // endpointURL := "https://s3..amazonaws.com" // endpointURL := "https://my-bucket.s3..amazonaws.com" // endpointURL := "https://s3..amazonaws.com/my-bucket" @@ -19,6 +22,7 @@ func ParseEndpoint(rawEndpoint string) (string, string, error) { } host := u.Hostname() parts := strings.SplitN(host, ".", 2) + fmt.Println("host", host, parts) var ( bucket string endpoint string diff --git a/server/api/studio/restapi/datasource.api b/server/api/studio/restapi/datasource.api index d77e8aba..a84713cf 100644 --- a/server/api/studio/restapi/datasource.api +++ b/server/api/studio/restapi/datasource.api @@ -3,10 +3,10 @@ syntax = "v1" type ( DatasourceS3Config { Endpoint string `json:"endpoint"` - Region string `json:"region"` + Region string `json:"region,optional"` Bucket string `json:"bucket"` AccessKey string `json:"accessKey"` - AccessSecret string `json:"accessSecret"` + AccessSecret string `json:"accessSecret,optional"` } DatasourceSFTPConfig { @@ -16,28 +16,30 @@ type ( Password string `json:"password"` } DatasourceS3UpdateConfig { - Endpoint string `json:"endpoint, optional, omitempty"` - Region string `json:"region, optional, omitempty"` - Bucket string `json:"bucket, optional, omitempty"` - AccessKey string `json:"accessKey, optional, omitempty"` - AccessSecret string `json:"accessSecret, optional, omitempty"` + Endpoint string `json:"endpoint,optional,omitempty"` + Region string `json:"region,optional,omitempty"` + Bucket string `json:"bucket,optional,omitempty"` + AccessKey string `json:"accessKey,optional,omitempty"` + AccessSecret string `json:"accessSecret,optional,omitempty"` } DatasourceSFTPUpdateConfig { - Host string `json:"host, optional, omitempty"` - Port int `json:"port, optional, omitempty"` - Username string `json:"username, optional, omitempty"` - Password string `json:"password, optional, omitempty"` + Host string `json:"host,optional,omitempty"` + Port int `json:"port,optional,omitempty"` + Username string `json:"username,optional,omitempty"` + Password string `json:"password,optional,omitempty"` } DatasourceAddRequest { Type string `json:"type"` + Platform string `json:"platform,optional,omitempty"` Name string `json:"name"` S3Config *DatasourceS3Config `json:"s3Config,optional"` SFTPConfig *DatasourceSFTPConfig `json:"sftpConfig,optional"` } DatasourceUpdateRequest { ID int `path:"id"` + Platform string `json:"platform,optional,omitempty"` Type string `json:"type"` Name string `json:"name"` S3Config *DatasourceS3UpdateConfig `json:"s3Config,optional"` @@ -63,6 +65,7 @@ type ( ID int `json:"id"` Type string `json:"type"` Name string `json:"name"` + Platform string `json:"platform"` S3Config *DatasourceS3Config `json:"s3Config,optional"` SFTPConfig *DatasourceSFTPConfig `json:"sftpConfig,optional"` CreateTime int64 `json:"createTime,optional"` diff --git a/server/api/studio/restapi/import.api b/server/api/studio/restapi/import.api index 0aaeb894..dc30f907 100644 --- a/server/api/studio/restapi/import.api +++ b/server/api/studio/restapi/import.api @@ -69,6 +69,14 @@ type ( Path string `json:"path,omitempty"` } + OSSConfig { + Endpoint string `json:"endpoint,omitempty"` + AccessKey string `json:"accessKey,omitempty"` + SecretKey string `json:"secretKey,omitempty"` + Bucket string `json:"bucket,omitempty"` + Key string `json:"key,omitempty"` + } + LocalConfig { Path string `json:"path,omitempty"` } @@ -104,6 +112,7 @@ type ( Path string `json:"path,optional,omitempty"` S3 *S3Config `json:"s3,optional,omitempty"` SFTP *SFTPConfig `json:"sftp,optional,omitempty"` + OSS *OSSConfig `json:"oss,optional,omitempty"` DatasourceId *int64 `json:"datasourceId,optional,omitempty"` DatasourceFilePath *string `json:"datasourceFilePath,optional,omitempty"` Tags []Tag `json:"tags,optional"` diff --git a/server/go.mod b/server/go.mod index 5d43afc8..1ff33f40 100644 --- a/server/go.mod +++ b/server/go.mod @@ -13,6 +13,7 @@ require ( require ( github.com/agiledragon/gomonkey/v2 v2.9.0 + github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible github.com/aws/aws-sdk-go v1.44.217 github.com/pkg/sftp v1.13.5 github.com/stretchr/testify v1.8.0 @@ -28,7 +29,6 @@ require ( ) require ( - github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/colinmarc/hdfs/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect From 76b6ad3cc9348c67f98784e26bb2bd6157db9428 Mon Sep 17 00:00:00 2001 From: hetao92 <18328704+hetao92@users.noreply.github.com> Date: Thu, 23 Mar 2023 11:04:57 +0000 Subject: [PATCH 10/11] mod: adjust the region rules in the config rules --- .../PlatformConfig/S3ConfigForm.tsx | 4 ++-- server/api/studio/internal/service/import.go | 18 +++++++++++++++--- server/api/studio/pkg/filestore/s3store.go | 6 +++++- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx index e3d04b9f..0c3cf9e3 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx @@ -42,9 +42,9 @@ const S3ConfigForm = (props: IProps) => { const platform = getFieldValue('platform'); return ( <> - {platform === IS3Platform.aws && + - } + .amazonaws.com' : intl.get('import.enterAddress')} /> diff --git a/server/api/studio/internal/service/import.go b/server/api/studio/internal/service/import.go index cfdae916..0ac1aa5d 100644 --- a/server/api/studio/internal/service/import.go +++ b/server/api/studio/internal/service/import.go @@ -91,7 +91,18 @@ func (i *importService) updateDatasourceConfig(conf *types.CreateImportTaskReque if err := json.Unmarshal([]byte(jsonConfig), cfg); err != nil { return nil, ecode.WithInternalServer(err, "get datasource config failed") } - if dbs.Platform == "oss" { + switch dbs.Platform { + case "aws": + // endpoint is not required in importer aws config + // some format of endpoint will cause error, for example: https://s3.amazonaws.com + source.S3 = &types.S3Config{ + AccessKey: cfg.AccessKey, + SecretKey: string(secret), + Bucket: cfg.Bucket, + Region: cfg.Region, + Key: *source.DatasourceFilePath, + } + case "oss": source.OSS = &types.OSSConfig{ AccessKey: cfg.AccessKey, SecretKey: string(secret), @@ -99,15 +110,16 @@ func (i *importService) updateDatasourceConfig(conf *types.CreateImportTaskReque Endpoint: cfg.Endpoint, Key: *source.DatasourceFilePath, } - } else { + case "customize": source.S3 = &types.S3Config{ AccessKey: cfg.AccessKey, SecretKey: string(secret), Bucket: cfg.Bucket, Region: cfg.Region, + Endpoint: cfg.Endpoint, Key: *source.DatasourceFilePath, } - if dbs.Platform == "customize" && cfg.Region == "" { + if cfg.Region == "" { source.S3.Region = "us-east-1" } } diff --git a/server/api/studio/pkg/filestore/s3store.go b/server/api/studio/pkg/filestore/s3store.go index 3dbf4da4..6219b004 100644 --- a/server/api/studio/pkg/filestore/s3store.go +++ b/server/api/studio/pkg/filestore/s3store.go @@ -30,8 +30,12 @@ func NewS3Store(platform, endpoint, region, bucket, accessKey, accessSecret stri Credentials: credentials.NewStaticCredentials(accessKey, accessSecret, ""), }) } else { + r := aws.String(region) + if region == "" { + r = aws.String("us-east-1") + } sess, err = session.NewSession(&aws.Config{ - Region: aws.String("us-east-1"), + Region: r, Credentials: credentials.NewStaticCredentials(accessKey, accessSecret, ""), S3ForcePathStyle: aws.Bool(true), Endpoint: aws.String(endpoint), From 6c9684ac5f42c2a7512fa096be9873a4d6e8138b Mon Sep 17 00:00:00 2001 From: hetao92 <18328704+hetao92@users.noreply.github.com> Date: Fri, 24 Mar 2023 07:33:31 +0000 Subject: [PATCH 11/11] feat: support cos platform --- app/config/locale/en-US.json | 7 +- app/config/locale/zh-CN.json | 7 +- app/interfaces/datasource.ts | 23 +++- .../PlatformConfig/S3ConfigForm.tsx | 18 ++- .../PlatformConfig/index.module.less | 9 ++ .../DatasourceConfig/PlatformConfig/index.tsx | 23 ++-- .../DatasourceList/RemoteList/index.tsx | 29 +++-- app/stores/datasource.ts | 7 - .../api/studio/internal/service/datasource.go | 52 +++----- server/api/studio/internal/service/import.go | 2 +- server/api/studio/pkg/filestore/filestore.go | 6 +- server/api/studio/pkg/filestore/ossStore.go | 122 ------------------ server/api/studio/pkg/filestore/s3store.go | 39 +++--- server/go.mod | 2 +- 14 files changed, 124 insertions(+), 222 deletions(-) delete mode 100644 server/api/studio/pkg/filestore/ossStore.go diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json index 5d19b4d3..dc8edb55 100644 --- a/app/config/locale/en-US.json +++ b/app/config/locale/en-US.json @@ -300,7 +300,12 @@ "preview": "Preview", "customize": "Customize", "s3Platform": "S3 Platform", - "account": "Account" + "account": "Account", + "endpointTip": "Please use an endpoint without a bucket name in the domain name, e.g. {sample}", + "awsTip": "https://s3.us-east-2.amazonaws.com", + "ossTip": "https://oss-cn-hangzhou.aliyuncs.com", + "cosTip": "https://cos.ap-shanghai.myqcloud.com", + "customizeTip": "http://127.0.0.1:9000" }, "schema": { "spaceList": "Graph Space List", diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json index 1913f01e..c46bbfb1 100644 --- a/app/config/locale/zh-CN.json +++ b/app/config/locale/zh-CN.json @@ -300,7 +300,12 @@ "directory": "目录", "preview": "预览", "customize": "自定义", - "s3Platform": "S3 平台" + "s3Platform": "S3 平台", + "endpointTip": "请使用域名中不含 bucket 名称的节点，例如 {sample}", + "awsTip": "https://s3.us-east-2.amazonaws.com", + "ossTip": "https://oss-cn-hangzhou.aliyuncs.com", + "cosTip": "https://cos.ap-shanghai.myqcloud.com", + "customizeTip": "http://127.0.0.1:9000" }, "schema": { "spaceList": "图空间列表", diff --git a/app/interfaces/datasource.ts b/app/interfaces/datasource.ts index fc01748b..62f04ea1 100644 --- a/app/interfaces/datasource.ts +++ b/app/interfaces/datasource.ts @@ -7,7 +7,7 @@ export enum IS3Platform { 'aws' = 'aws', 'oss' = 'oss', 'customize' = 'customize', - 'tecent' = 'tecent' + 'tecent' = 'cos' } export interface IDatasourceAdd { @@ -30,4 +30,25 @@ export interface IDatasourceAdd { } export interface IDatasourceUpdate extends IDatasourceAdd { id: number; +} + +export interface IDatasourceItem { + id?: number; + name: string; + type: IDatasourceType; + platform?: string; + createTime?: string; + s3Config?: { + accessKey: string; + accessSecret?: string; + endpoint: string; + bucket: string; + region?: string; + }; + sftpConfig?: { + host: string; + port: number; + username: string; + password: string; + }; } \ No newline at end of file diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx index 0c3cf9e3..f8831db1 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/S3ConfigForm.tsx @@ -1,8 +1,10 @@ import { useI18n } from '@vesoft-inc/i18n'; import { Input, Form, Select, FormInstance } from 'antd'; -import React, { useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { observer } from 'mobx-react-lite'; import { IS3Platform } from '@app/interfaces/datasource'; +import Instruction from '@app/components/Instruction'; +import styles from './index.module.less'; const FormItem = Form.Item; interface IProps { formRef: FormInstance; @@ -27,10 +29,13 @@ const S3ConfigForm = (props: IProps) => { } }; + const handleReset = useCallback(() => { + formRef.resetFields(['s3Config']); + }, []); return ( - AWS S3 Aliyun OSS Tecent COS @@ -45,8 +50,13 @@ const S3ConfigForm = (props: IProps) => { - - .amazonaws.com' : intl.get('import.enterAddress')} /> + +
+ + .amazonaws.com' : intl.get('import.enterAddress')} /> + + {platform && } +
diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less index 39e7418b..58079b3c 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.module.less @@ -21,5 +21,14 @@ margin-bottom: 20px; text-align: center; } + .endpointFormItem { + position: relative; + } + :global(.studioIconInstruction) { + position: absolute; + top: 50%; + right: 115px; + transform: translateY(-50%); + } } diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx index b4620c2e..94f9d220 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/PlatformConfig/index.tsx @@ -1,7 +1,7 @@ import { useI18n } from '@vesoft-inc/i18n'; import { Button, Modal, Form, Select, message } from 'antd'; import React, { useMemo, useState } from 'react'; -import { IDatasourceType } from '@app/interfaces/datasource'; +import { IDatasourceItem, IDatasourceType } from '@app/interfaces/datasource'; import { v4 as uuidv4 } from 'uuid'; import { useStore } from '@app/stores'; import { observer } from 'mobx-react-lite'; @@ -16,7 +16,7 @@ interface IProps { onConfirm: () => void; onCancel: () => void; type?: IDatasourceType; - data?: any; + data?: IDatasourceItem; } const fomrItemLayout = { @@ -33,7 +33,7 @@ const DatasourceConfigModal = (props: IProps) => { const [loading, setLoading] = useState(false); const tempPwd = useMemo(() => uuidv4() + Date.now(), []); const mode = useMemo(() => (data ? 'edit' : 'create'), [data]); - const submit = async (values: any) => { + const submit = async (values: IDatasourceItem) => { const _type = values.type || type; setLoading(true); if(mode === 'create') { @@ -45,11 +45,18 @@ const DatasourceConfigModal = (props: IProps) => { setLoading(false); flag && (message.success(intl.get('schema.createSuccess')), onConfirm()); } else { - const _config = _type === IDatasourceType.s3 ? 's3Config' : 'sftpConfig'; - if(_type === IDatasourceType.s3 && values[_config].accessSecret === tempPwd) { - delete values[_config].accessSecret; - } else if (_type === IDatasourceType.sftp && values[_config].password === tempPwd) { - delete values[_config].password; + let _config; + switch (_type) { + case IDatasourceType.s3: + _config = values.s3Config; + _config.accessSecret === tempPwd && delete _config.accessSecret; + break; + case IDatasourceType.sftp: + _config = values.sftpConfig; + _config.password === tempPwd && delete _config.password; + break; + default: + break; } const flag = await updateDataSource({ id: data.id, diff --git a/app/pages/Import/DatasourceList/RemoteList/index.tsx b/app/pages/Import/DatasourceList/RemoteList/index.tsx index 3017199c..85f3f5c5 100644 --- a/app/pages/Import/DatasourceList/RemoteList/index.tsx +++ b/app/pages/Import/DatasourceList/RemoteList/index.tsx @@ -5,7 +5,7 @@ import { trackPageView } from '@app/utils/stat'; import { Button, message, Popconfirm, Table } from 'antd'; import Icon from '@app/components/Icon'; import cls from 'classnames'; -import { IDatasourceType } from '@app/interfaces/datasource'; +import { IDatasourceType, IDatasourceItem } from '@app/interfaces/datasource'; import { useI18n } from '@vesoft-inc/i18n'; import dayjs from 'dayjs'; @@ -62,18 +62,18 @@ const DatasourceList = (props: IProps) => { const { type } = props; const { datasource } = useStore(); const { intl } = useI18n(); - const { getDatasourceList, datasourceList, deleteDataSource, batchDeleteDatasource } = datasource; - const [data, setData] = useState([]); - const [editData, setEditData] = useState(null); + const { getDatasourceList, deleteDataSource, batchDeleteDatasource } = datasource; + const [data, setData] = useState([]); + const [editData, setEditData] = useState(null); const [loading, setLoading] = useState(false); const [visible, setVisible] = useState(false); - const [selectItems, setSelectItems] = useState([]); + const [selectIds, setSelectIds] = useState([]); const create = useCallback(() => { setEditData(null); setVisible(true); }, []); - const editItem = useCallback((item) => { + const editItem = useCallback((item: IDatasourceItem) => { setEditData(item); setVisible(true); }, []); @@ -88,7 +88,7 @@ const DatasourceList = (props: IProps) => { })).concat(({ title: intl.get('common.operation'), key: 'operation', - render: (_, item) => (
+ render: (_, item: IDatasourceItem) => (
@@ -113,12 +113,12 @@ const DatasourceList = (props: IProps) => { setLoading(false); }; const handleDeleteDatasource = useCallback(async () => { - const flag = await batchDeleteDatasource(selectItems); + const flag = await batchDeleteDatasource(selectIds); if (!flag) return; message.success(intl.get('common.deleteSuccess')); getList(); - setSelectItems([]); - }, [selectItems]); + setSelectIds([]); + }, [selectIds]); const handleRefresh = () => { getList(); @@ -141,21 +141,22 @@ const DatasourceList = (props: IProps) => { title={intl.get('common.ask')} okText={intl.get('common.confirm')} cancelText={intl.get('common.cancel')} - disabled={!selectItems.length} + disabled={!selectIds.length} > -
-

{intl.get('import.datasourceList', { type: intl.get(`import.${type}`) })} ({datasourceList.length})

+

{intl.get('import.datasourceList', { type: intl.get(`import.${type}`) })} ({data.length})

setSelectItems(selectedRowKeys as number[]), + selectedRowKeys: selectIds, + onChange: (selectedRowKeys) => setSelectIds(selectedRowKeys as number[]), }} columns={columns} rowKey="id" diff --git a/app/stores/datasource.ts b/app/stores/datasource.ts index 19b9a9ec..16179325 100644 --- a/app/stores/datasource.ts +++ b/app/stores/datasource.ts @@ -1,15 +1,8 @@ -import { makeAutoObservable, observable } from 'mobx'; import service from '@app/config/service'; import { IDatasourceAdd, IDatasourceType, IDatasourceUpdate } from '@app/interfaces/datasource'; import { getRootStore } from '.'; export class DatasourceStore { - datasourceList = []; - constructor() { - makeAutoObservable(this, { - datasourceList: observable, - }); - } get rootStore() { return getRootStore(); diff --git a/server/api/studio/internal/service/datasource.go b/server/api/studio/internal/service/datasource.go index a6680f94..9ab3c9b1 100644 --- a/server/api/studio/internal/service/datasource.go +++ b/server/api/studio/internal/service/datasource.go @@ -330,35 +330,23 @@ func formatDatasourceConfig(config interface{}, password string) (string, string func validate(typ string, platform string, config interface{}) (string, string, error) { switch typ { case "s3": - if platform == "oss" { - cfg := config.(*types.DatasourceS3Config) - err := validateOss(cfg) - if err != nil { - return "", "", err - } - secret := cfg.AccessSecret - cfg.AccessSecret = "" - cfgStr, crypto, err := formatDatasourceConfig(config, secret) - return cfgStr, crypto, err - } else { - cfg := config.(*types.DatasourceS3Config) - endpoint, parsedBucket, err := utils.ParseEndpoint(platform, cfg.Endpoint) - if err != nil { - return "", "", err - } - if parsedBucket != "" && cfg.Bucket != parsedBucket { - return "", "", errors.New("bucket name in endpoint and bucket name in config are different") - } - cfg.Endpoint = endpoint - err = validateS3(platform, cfg) - if err != nil { - return "", "", err - } - secret := cfg.AccessSecret - cfg.AccessSecret = "" - cfgStr, crypto, err := formatDatasourceConfig(config, secret) - return cfgStr, crypto, err + cfg := config.(*types.DatasourceS3Config) + endpoint, parsedBucket, err := utils.ParseEndpoint(platform, cfg.Endpoint) + if err != nil { + return "", "", err + } + if parsedBucket != "" && cfg.Bucket != parsedBucket { + return "", "", errors.New("bucket name in endpoint and bucket name in config are different") } + cfg.Endpoint = endpoint + err = validateS3(platform, cfg) + if err != nil { + return "", "", err + } + secret := cfg.AccessSecret + cfg.AccessSecret = "" + cfgStr, crypto, err := formatDatasourceConfig(config, secret) + return cfgStr, crypto, err case "sftp": cfg := config.(*types.DatasourceSFTPConfig) err := validateSftp(cfg) @@ -374,14 +362,6 @@ func validate(typ string, platform string, config interface{}) (string, string, } } -func validateOss(cfg *types.DatasourceS3Config) error { - _, err := filestore.NewOssStore(cfg.Endpoint, cfg.Bucket, cfg.AccessKey, cfg.AccessSecret) - if err != nil { - return fmt.Errorf("connect the oss client error: %s", err) - } - return nil -} - func validateSftp(cfg *types.DatasourceSFTPConfig) error { store, err := filestore.NewSftpStore(cfg.Host, cfg.Port, cfg.Username, cfg.Password) if err != nil { diff --git a/server/api/studio/internal/service/import.go b/server/api/studio/internal/service/import.go index 0ac1aa5d..903efc1b 100644 --- a/server/api/studio/internal/service/import.go +++ b/server/api/studio/internal/service/import.go @@ -110,7 +110,7 @@ func (i *importService) updateDatasourceConfig(conf *types.CreateImportTaskReque Endpoint: cfg.Endpoint, Key: *source.DatasourceFilePath, } - case "customize": + case "cos", "customize": source.S3 = &types.S3Config{ AccessKey: cfg.AccessKey, SecretKey: string(secret), diff --git a/server/api/studio/pkg/filestore/filestore.go b/server/api/studio/pkg/filestore/filestore.go index 808c6e04..3f70fadb 100644 --- a/server/api/studio/pkg/filestore/filestore.go +++ b/server/api/studio/pkg/filestore/filestore.go @@ -40,11 +40,7 @@ func NewFileStore(typ, config, secret, platform string) (FileStore, error) { if err := json.Unmarshal([]byte(config), &c); err != nil { return nil, errors.New("parse the s3 config error") } - if platform == "oss" { - return NewOssStore(c.Endpoint, c.Bucket, c.AccessKey, secret) - } else { - return NewS3Store(platform, c.Endpoint, c.Region, c.Bucket, c.AccessKey, secret) - } + return NewS3Store(platform, c.Endpoint, c.Region, c.Bucket, c.AccessKey, secret) case "sftp": var c SftpConfig if err := json.Unmarshal([]byte(config), &c); err != nil { diff --git a/server/api/studio/pkg/filestore/ossStore.go b/server/api/studio/pkg/filestore/ossStore.go deleted file mode 100644 index d250eead..00000000 --- a/server/api/studio/pkg/filestore/ossStore.go +++ /dev/null @@ -1,122 +0,0 @@ -package filestore - -import ( - "bufio" - "errors" - "fmt" - "strings" - - "github.com/aliyun/aliyun-oss-go-sdk/oss" -) - -type OssStore struct { - Client *oss.Client - Bucket *oss.Bucket -} - -func NewOssStore(endpoint, bucketName, accessKey, accessSecret string) (*OssStore, error) { - client, err := oss.New(endpoint, accessKey, accessSecret) - if err != nil { - return nil, fmt.Errorf("failed to create oss client: %v", err) - } - bucket, err := client.Bucket(bucketName) - if err != nil { - return nil, fmt.Errorf("failed to get bucket: %v", err) - } - return &OssStore{ - Client: client, - Bucket: bucket, - }, nil -} - -func (s *OssStore) ReadFile(path string, startLine ...int) ([]string, error) { - var numLines int - var start int - if len(startLine) == 0 { - start = 0 - numLines = -1 - } else if len(startLine) == 1 { - start = startLine[0] - numLines = -1 - } else { - start = startLine[0] - numLines = startLine[1] - } - - resp, err := s.Bucket.GetObject(path) - if err != nil { - return nil, err - } - defer resp.Close() - - fileScanner := bufio.NewScanner(resp) - - var lines []string - for i := 0; i < start; i++ { - if !fileScanner.Scan() { - return nil, errors.New("start line is beyond end of file") - } - } - - for i := 0; numLines < 0 || i < numLines; i++ { - if !fileScanner.Scan() { - break - } - lines = append(lines, fileScanner.Text()) - } - - if err := fileScanner.Err(); err != nil { - return nil, err - } - - return lines, nil -} - -func (s *OssStore) ListFiles(path string) ([]FileConfig, error) { - resp, err := s.Bucket.ListObjectsV2(oss.Prefix(path), oss.Delimiter("/")) - if err != nil { - return nil, err - } - var files []FileConfig - - for _, str := range resp.CommonPrefixes { - name := str[:len(str)-1] // remove trailing slash - files = append(files, FileConfig{ - Name: strings.TrimPrefix(name, path), - Type: "directory", - }) - } - for _, obj := range resp.Objects { - var objType string - key := obj.Key - if key[len(key)-1:] == "/" { - objType = "directory" - } else if strings.HasSuffix(key, ".csv") { - objType = "csv" - } - name := strings.TrimPrefix(key, path) - if objType != "" && name != "" { - s3Object := FileConfig{ - Name: name, - Type: objType, - Size: obj.Size, - } - files = append(files, s3Object) - } - } - - return files, nil -} - -func (s *OssStore) ListBuckets() ([]string, error) { - resp, err := s.Client.ListBuckets() - if err != nil { - return nil, err - } - var buckets []string - for _, obj := range resp.Buckets { - buckets = append(buckets, obj.Name) - } - - return buckets, nil -} diff --git a/server/api/studio/pkg/filestore/s3store.go b/server/api/studio/pkg/filestore/s3store.go index 6219b004..5e39b3ed 100644 --- a/server/api/studio/pkg/filestore/s3store.go +++ b/server/api/studio/pkg/filestore/s3store.go @@ -21,26 +21,22 @@ type S3Store struct { } func NewS3Store(platform, endpoint, region, bucket, accessKey, accessSecret string) (*S3Store, error) { - var sess *session.Session - var err error - - if platform == "aws" { - sess, err = session.NewSession(&aws.Config{ - Region: aws.String(region), - Credentials: credentials.NewStaticCredentials(accessKey, accessSecret, ""), - }) - } else { - r := aws.String(region) - if region == "" { - r = aws.String("us-east-1") - } - sess, err = session.NewSession(&aws.Config{ - Region: r, - Credentials: credentials.NewStaticCredentials(accessKey, accessSecret, ""), - S3ForcePathStyle: aws.Bool(true), - Endpoint: aws.String(endpoint), - }) + cfg := &aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewStaticCredentials(accessKey, accessSecret, ""), + } + switch platform { + case "oss": + cfg.S3ForcePathStyle = aws.Bool(false) + cfg.Endpoint = aws.String(endpoint) + case "cos", "customize": + cfg.S3ForcePathStyle = aws.Bool(true) + cfg.Endpoint = aws.String(endpoint) + } + if region == "" { + cfg.Region = aws.String("us-east-1") } + sess, err := session.NewSession(cfg) if err != nil { return nil, errors.New("failed to create session") } @@ -134,9 +130,10 @@ func (s *S3Store) ListFiles(s3path string) ([]FileConfig, error) { } else if strings.HasSuffix(key, ".csv") { objType = "csv" } - if objType != "" { + name := strings.TrimPrefix(key, s3path) + if objType != "" && name != "" { s3Object := FileConfig{ - Name: strings.TrimPrefix(key, s3path), + Name: name, Type: objType, Size: *obj.Size, } diff --git a/server/go.mod b/server/go.mod index 1ff33f40..5d43afc8 100644 --- a/server/go.mod +++ b/server/go.mod @@ -13,7 +13,6 @@ require ( require ( github.com/agiledragon/gomonkey/v2 v2.9.0 - github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible github.com/aws/aws-sdk-go v1.44.217 github.com/pkg/sftp v1.13.5 github.com/stretchr/testify v1.8.0 @@ -29,6 +28,7 @@ require ( ) require ( + github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/colinmarc/hdfs/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect