From eb8e692f927a79405db87e8d782aece883d95172 Mon Sep 17 00:00:00 2001 From: Nut He <18328704+hetao92@users.noreply.github.com> Date: Thu, 26 May 2022 15:07:49 +0800 Subject: [PATCH] refactor: server update & bugfix (#230) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat: mime && connectdb feat: 文件上传、删除、预览功能实现 (#206) feat: middleware (#208) * feat: middleware * feat: dynamicly add query params func * feat: common params * update: go mod tidy * feat: batchExec complete file and import task (#213) modify upload and download mody: go mod refine the code update: make check modify return values Refactor (#219) * update: server-v2 * updata: format feat: context auth (#220) update: export module (#222) feat: export RegisterHandlers (#223) update: gitignore (#224) fix: fix service init crash (#225) update: WithErrorMessage (#226) fix import task (#227) fix the type conveersion problem fix: fix import & file model (#229) Co-authored-by: Bruce --- .gitignore | 10 +- app/config/service.ts | 60 +- app/interfaces/import.ts | 12 +- app/pages/Import/FileUpload/index.tsx | 32 +- .../Import/TaskCreate/FileSelect/index.tsx | 14 +- .../TaskCreate/PasswordInputModal/index.tsx | 2 +- app/pages/Import/TaskCreate/index.tsx | 30 +- .../TaskList/TaskItem/LogModal/index.tsx | 20 +- app/pages/Import/TaskList/TaskItem/index.tsx | 52 +- .../Import/TaskList/TemplateModal/index.tsx | 14 +- app/pages/Import/TaskList/index.tsx | 10 +- app/pages/Import/index.tsx | 5 +- app/stores/files.ts | 15 +- app/stores/import.ts | 80 +- app/utils/http.ts | 38 +- app/utils/import.ts | 14 +- config/server.js | 4 +- config/webpack.prod.js | 2 +- server-v2/.golangci.yml | 263 ++++++ server-v2/Makefile | 83 ++ server-v2/api/studio/.goctl/api/handler.tpl | 31 + server-v2/api/studio/Dockerfile | 33 + server-v2/api/studio/Makefile | 107 +++ server-v2/api/studio/assets/.gitkeep | 0 server-v2/api/studio/etc/studio-api.yaml | 14 + .../api/studio/internal/config/config.go | 112 +++ .../handler/file/filedestroyhandler.go | 33 + .../handler/file/filesindexhandler.go | 17 + .../handler/file/fileuploadhandler.go | 17 + .../handler/gateway/batchexecngqlhandler.go | 33 + .../handler/gateway/connecthandler.go | 33 + .../handler/gateway/disonnecthandler.go | 17 + .../handler/gateway/execngqlhandler.go | 33 + .../internal/handler/health/gethandler.go | 17 + .../importtask/createimporttaskhandler.go | 33 + .../importtask/deleteimporttaskhandler.go | 33 + .../importtask/downloadconfighandler.go | 33 + .../handler/importtask/downloadlogshandler.go | 33 + .../importtask/getimporttaskhandler.go | 33 + .../getimporttasklognameshandler.go | 33 + .../importtask/getmanyimporttaskhandler.go | 33 + .../importtask/getmanyimporttaskloghandler.go | 33 + .../importtask/getworkingdirhandler.go | 17 + .../importtask/stopimporttaskhandler.go | 33 + .../api/studio/internal/handler/routes.go | 133 +++ .../internal/logic/file/filedestroylogic.go | 30 + .../internal/logic/file/filesindexlogic.go | 29 + .../internal/logic/file/fileuploadlogic.go | 27 + .../logic/gateway/batchexecngqllogic.go | 29 + .../internal/logic/gateway/connectlogic.go | 29 + .../internal/logic/gateway/disonnectlogic.go | 29 + .../internal/logic/gateway/execngqllogic.go | 29 + .../studio/internal/logic/health/getlogic.go | 29 + .../logic/importtask/createimporttasklogic.go | 29 + .../logic/importtask/deleteimporttasklogic.go | 29 + .../logic/importtask/downloadconfiglogic.go | 30 + .../logic/importtask/downloadlogslogic.go | 30 + .../logic/importtask/getimporttasklogic.go | 29 + .../importtask/getimporttasklognameslogic.go | 29 + .../importtask/getmanyimporttasklogic.go | 29 + .../importtask/getmanyimporttaskloglogic.go | 30 + .../logic/importtask/getworkingdirlogic.go | 29 + .../logic/importtask/stopimporttasklogic.go | 30 + server-v2/api/studio/internal/service/file.go | 265 ++++++ .../api/studio/internal/service/gateway.go | 157 ++++ .../api/studio/internal/service/health.go | 38 + .../api/studio/internal/service/import.go | 331 +++++++ .../internal/service/importer/importer.go | 250 ++++++ .../studio/internal/service/importer/task.go | 21 + .../internal/service/importer/taskInfo.go | 16 + .../internal/service/importer/taskdb.go | 87 ++ .../internal/service/importer/taskmgr.go | 269 ++++++ .../api/studio/internal/svc/servicecontext.go | 60 ++ server-v2/api/studio/internal/types/types.go | 238 +++++ server-v2/api/studio/pkg/auth/authorize.go | 153 ++++ server-v2/api/studio/pkg/base/types.go | 10 + server-v2/api/studio/pkg/ecode/alias.go | 52 ++ server-v2/api/studio/pkg/ecode/codes.go | 106 +++ server-v2/api/studio/pkg/logging/logging.go | 49 ++ server-v2/api/studio/pkg/server/server.go | 24 + server-v2/api/studio/pkg/utils/file.go | 13 + server-v2/api/studio/pkg/utils/http.go | 89 ++ server-v2/api/studio/restapi/file.api | 35 + server-v2/api/studio/restapi/gateway.api | 53 ++ server-v2/api/studio/restapi/health.api | 17 + server-v2/api/studio/restapi/import.api | 241 +++++ server-v2/api/studio/restapi/studio.api | 13 + server-v2/api/studio/studio.go | 77 ++ server-v2/go.mod | 67 ++ server-v2/go.sum | 831 ++++++++++++++++++ 90 files changed, 5549 insertions(+), 242 deletions(-) create mode 100644 server-v2/.golangci.yml create mode 100644 server-v2/Makefile create mode 100644 server-v2/api/studio/.goctl/api/handler.tpl create mode 100644 server-v2/api/studio/Dockerfile create mode 100644 server-v2/api/studio/Makefile create mode 100644 server-v2/api/studio/assets/.gitkeep create mode 100644 server-v2/api/studio/etc/studio-api.yaml create mode 100644 server-v2/api/studio/internal/config/config.go create mode 100644 server-v2/api/studio/internal/handler/file/filedestroyhandler.go create mode 100644 server-v2/api/studio/internal/handler/file/filesindexhandler.go create mode 100644 server-v2/api/studio/internal/handler/file/fileuploadhandler.go create mode 100644 server-v2/api/studio/internal/handler/gateway/batchexecngqlhandler.go create mode 100644 server-v2/api/studio/internal/handler/gateway/connecthandler.go create mode 100644 server-v2/api/studio/internal/handler/gateway/disonnecthandler.go create mode 100644 server-v2/api/studio/internal/handler/gateway/execngqlhandler.go create mode 100644 server-v2/api/studio/internal/handler/health/gethandler.go create mode 100644 server-v2/api/studio/internal/handler/importtask/createimporttaskhandler.go create mode 100644 server-v2/api/studio/internal/handler/importtask/deleteimporttaskhandler.go create mode 100644 server-v2/api/studio/internal/handler/importtask/downloadconfighandler.go create mode 100644 server-v2/api/studio/internal/handler/importtask/downloadlogshandler.go create mode 100644 server-v2/api/studio/internal/handler/importtask/getimporttaskhandler.go create mode 100644 server-v2/api/studio/internal/handler/importtask/getimporttasklognameshandler.go create mode 100644 server-v2/api/studio/internal/handler/importtask/getmanyimporttaskhandler.go create mode 100644 server-v2/api/studio/internal/handler/importtask/getmanyimporttaskloghandler.go create mode 100644 server-v2/api/studio/internal/handler/importtask/getworkingdirhandler.go create mode 100644 server-v2/api/studio/internal/handler/importtask/stopimporttaskhandler.go create mode 100644 server-v2/api/studio/internal/handler/routes.go create mode 100644 server-v2/api/studio/internal/logic/file/filedestroylogic.go create mode 100644 server-v2/api/studio/internal/logic/file/filesindexlogic.go create mode 100644 server-v2/api/studio/internal/logic/file/fileuploadlogic.go create mode 100644 server-v2/api/studio/internal/logic/gateway/batchexecngqllogic.go create mode 100644 server-v2/api/studio/internal/logic/gateway/connectlogic.go create mode 100644 server-v2/api/studio/internal/logic/gateway/disonnectlogic.go create mode 100644 server-v2/api/studio/internal/logic/gateway/execngqllogic.go create mode 100644 server-v2/api/studio/internal/logic/health/getlogic.go create mode 100644 server-v2/api/studio/internal/logic/importtask/createimporttasklogic.go create mode 100644 server-v2/api/studio/internal/logic/importtask/deleteimporttasklogic.go create mode 100644 server-v2/api/studio/internal/logic/importtask/downloadconfiglogic.go create mode 100644 server-v2/api/studio/internal/logic/importtask/downloadlogslogic.go create mode 100644 server-v2/api/studio/internal/logic/importtask/getimporttasklogic.go create mode 100644 server-v2/api/studio/internal/logic/importtask/getimporttasklognameslogic.go create mode 100644 server-v2/api/studio/internal/logic/importtask/getmanyimporttasklogic.go create mode 100644 server-v2/api/studio/internal/logic/importtask/getmanyimporttaskloglogic.go create mode 100644 server-v2/api/studio/internal/logic/importtask/getworkingdirlogic.go create mode 100644 server-v2/api/studio/internal/logic/importtask/stopimporttasklogic.go create mode 100644 server-v2/api/studio/internal/service/file.go create mode 100644 server-v2/api/studio/internal/service/gateway.go create mode 100644 server-v2/api/studio/internal/service/health.go create mode 100644 server-v2/api/studio/internal/service/import.go create mode 100644 server-v2/api/studio/internal/service/importer/importer.go create mode 100644 server-v2/api/studio/internal/service/importer/task.go create mode 100644 server-v2/api/studio/internal/service/importer/taskInfo.go create mode 100644 server-v2/api/studio/internal/service/importer/taskdb.go create mode 100644 server-v2/api/studio/internal/service/importer/taskmgr.go create mode 100644 server-v2/api/studio/internal/svc/servicecontext.go create mode 100644 server-v2/api/studio/internal/types/types.go create mode 100644 server-v2/api/studio/pkg/auth/authorize.go create mode 100644 server-v2/api/studio/pkg/base/types.go create mode 100644 server-v2/api/studio/pkg/ecode/alias.go create mode 100644 server-v2/api/studio/pkg/ecode/codes.go create mode 100644 server-v2/api/studio/pkg/logging/logging.go create mode 100644 server-v2/api/studio/pkg/server/server.go create mode 100644 server-v2/api/studio/pkg/utils/file.go create mode 100644 server-v2/api/studio/pkg/utils/http.go create mode 100644 server-v2/api/studio/restapi/file.api create mode 100644 server-v2/api/studio/restapi/gateway.api create mode 100644 server-v2/api/studio/restapi/health.api create mode 100644 server-v2/api/studio/restapi/import.api create mode 100644 server-v2/api/studio/restapi/studio.api create mode 100644 server-v2/api/studio/studio.go create mode 100644 server-v2/go.mod create mode 100644 server-v2/go.sum diff --git a/.gitignore b/.gitignore index a5f89fe7..ca0debd0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,10 +5,14 @@ node_modules/ .DS_Store .vscode *.swp +*.swo +*.out +*.log *.lock *.map .github/workflows/nodejs.yml + app/**/*.map config/**/*.map dist/ @@ -19,5 +23,9 @@ dist/ !app/**/*.js tmp server/data -assets/ +server/assets/* +!server/assets/.gitkeep +server-v2/api/*/data +server-v2/api/*/assets/* +!server-v2/api/*/assets/.gitkeep bin/ \ No newline at end of file diff --git a/app/config/service.ts b/app/config/service.ts index 56e6c46f..909c7974 100644 --- a/app/config/service.ts +++ b/app/config/service.ts @@ -1,61 +1,57 @@ import { _delete, get, post, put } from '../utils/http'; -let service = { - execNGQL: (params, config?) => { - return post('/api-nebula/db/exec')(params, config) +const service = { + execNGQL: (params, config?) => { + return post('/api-nebula/db/exec')(params, config); }, batchExecNGQL: (params, config?) => { - return post('/api-nebula/db/batchExec')(params, config) + return post('/api-nebula/db/batchExec')(params, config); }, connectDB: (params, config?) => { - return post('/api-nebula/db/connect')(params, config) + return post('/api-nebula/db/connect')(params, config); }, disconnectDB: (params, config?) => { - return post('/api-nebula/db/disconnect')(params, config) + return post('/api-nebula/db/disconnect')(params, config); }, + // import importData: (params, config?) => { - return post('/api/import-tasks/import')(params, config) + return post('/api/import-tasks')(params, config); }, - handleImportAction: (params, config?) => { - return post('/api/import-tasks/action')(params, config) + stopImportTask: (id: number) => { + return post(`/api/import-tasks/${id}/stop`)(); }, - getLog: (params, config?) => { - return get('/api/import-tasks/logs')(params, config) + deleteImportTask: (id: number) => { + return post(`/api/import-tasks/${id}/stop`)(); }, - getErrLog: (params, config?) => { - return get('/api/import-tasks/err-logs')(params, config) + getTaskList: (params?, config?) => { + return get('/api/import-tasks')(params, config); }, - finishImport: (params, config?) => { - return post('/api/import/finish')(params, config) - }, - getUploadDir: () => { - return get('/api/import-tasks/working-dir')() + getTaskLogs: (params?, config?) => { + const { id, ...others } = params; + return get(`/api/import-tasks/${id}/task-log-names`)(others, config); }, - getTaskDir: () => { - return get('/api/import-tasks/task-dir')() + getLogDetail: (params, config?) => { + const { id, ...others } = params; + return get(`/api/import-tasks/${id}/logs`)(others, config); }, - deteleFile:params => { + getTaskConfig: (id: string | number) => `/api/import-tasks/${id}/download-config`, + getTaskLog: (id: string | number) => `/api/import-tasks/${id}/download-logs`, + // files + deteleFile: params => { const { filename } = params; return _delete(`/api/files/${filename}`)(); }, getFiles: () => { - return get('/api/files')() + return get('/api/files')(); }, uploadFiles: (params?, config?) => { put('/api/files')(params, { ...config, headers: { 'Content-Type': 'multipart/form-data' } }); }, - getTaskLogs: (params?, config?) => { - const { id, ...others } = params; - return get(`/api/import-tasks/${id}/task-log-names`)(others, config); - }, - getTaskConfigUrl: (id: string | number) => `/api/import-tasks/config/${id}`, - getTaskLogUrl: (id: string | number) => `/api/import-tasks/${id}/log`, - getTaskErrLogUrl: (id: string | number) => `/api/import-tasks/${id}/err-logs`, -} +}; export const updateService = (partService: any) => { - Object.assign(service, partService) -} + Object.assign(service, partService); +}; export default service; diff --git a/app/interfaces/import.ts b/app/interfaces/import.ts index 222032be..58e67825 100644 --- a/app/interfaces/import.ts +++ b/app/interfaces/import.ts @@ -18,15 +18,15 @@ export interface ITaskStats { numReadFailed: number; } export interface ITaskItem { - taskID: number; - nebulaAddress: string; + id: number; + address: string; space: string; name: string; - createdTime: number; - updatedTime: number; + createTime: number; + updateTime: number; user: string; - taskStatus: ITaskStatus; - taskMessage: string; + status: ITaskStatus; + message: string; stats: ITaskStats; } diff --git a/app/pages/Import/FileUpload/index.tsx b/app/pages/Import/FileUpload/index.tsx index a5b09462..887fe13b 100644 --- a/app/pages/Import/FileUpload/index.tsx +++ b/app/pages/Import/FileUpload/index.tsx @@ -2,30 +2,23 @@ import React, { useEffect, useState } from 'react'; import { observer } from 'mobx-react-lite'; import { useStore } from '@app/stores'; import { trackPageView } from '@app/utils/stat'; -import FileList from './FileList'; import { debounce } from 'lodash'; import { message } from 'antd'; import intl from 'react-intl-universal'; import { StudioFile } from '@app/interfaces/import'; -interface IProps { - needFileDir: boolean -} +import FileList from './FileList'; -const FileUpload = (props: IProps) => { +const FileUpload = () => { const { files } = useStore(); - const { fileList, uploadDir, deleteFile, getFiles, uploadFile, getUploadDir } = files; + const { fileList, deleteFile, getFiles, uploadFile } = files; const [loading, setLoading] = useState(false); - const { needFileDir = true } = props; const transformFile = async (_file: StudioFile, fileList: StudioFile[]) => { fileList.forEach(file => { - if(needFileDir) { - file.path = `${uploadDir}/${file.name}`; - } + file.path = `${file.name}`; file.withHeader = false; - }) - - await handleUpdate(fileList) - return false + }); + await handleUpdate(fileList); + return false; }; const handleUpdate = async (fileList: StudioFile[]) => { @@ -33,8 +26,8 @@ const FileUpload = (props: IProps) => { await uploadFile(fileList).then(_ => { setTimeout(() => { getFileList(); - message.success(intl.get('import.uploadSuccessfully')) - }, 2000) + message.success(intl.get('import.uploadSuccessfully')); + }, 2000); }).catch(_err => { setLoading(false); }); @@ -42,17 +35,16 @@ const FileUpload = (props: IProps) => { const handleDelete = (index: number) => { const file = fileList[index].name; - deleteFile(file) - } + deleteFile(file); + }; const getFileList = async () => { !loading && setLoading(true); await getFiles(); setLoading(false); - } + }; useEffect(() => { getFileList(); - needFileDir && getUploadDir(); trackPageView('/import/files'); }, []); return ( diff --git a/app/pages/Import/TaskCreate/FileSelect/index.tsx b/app/pages/Import/TaskCreate/FileSelect/index.tsx index a7260a84..bef9a985 100644 --- a/app/pages/Import/TaskCreate/FileSelect/index.tsx +++ b/app/pages/Import/TaskCreate/FileSelect/index.tsx @@ -1,5 +1,5 @@ import { Button, Form, Popover, Select } from 'antd'; -import React, { useEffect, useState } from 'react'; +import React, { useState } from 'react'; import intl from 'react-intl-universal'; import { observer } from 'mobx-react-lite'; import { useStore } from '@app/stores'; @@ -42,9 +42,13 @@ const FileSelect = (props: IProps) => { } setVisible(false); }; - useEffect(() => { - getFiles(); - }, []); + + const handleGetFiles = () => { + if(fileList.length === 0) { + getFiles(); + } + }; + return ( { onVisibleChange={visible => setVisible(visible)} content={
- {fileList.map((file: any) => (