From ab4ef8b5a48c6eaa9f45285229d3cf8ee98d024e Mon Sep 17 00:00:00 2001 From: jsonwan Date: Mon, 19 Jul 2021 17:39:17 +0800 Subject: [PATCH] =?UTF-8?q?feature:=20=E6=8F=90=E4=BE=9B=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=BA=90=E7=AE=A1=E7=90=86=E3=80=81=E5=87=AD=E6=8D=AE=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E7=9B=B8=E5=85=B3ESB=20API=20#92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../i18n/exception/message.properties | 7 +- .../i18n/exception/message_en.properties | 7 +- .../i18n/exception/message_en_US.properties | 7 +- .../i18n/exception/message_zh.properties | 7 +- .../i18n/exception/message_zh_CN.properties | 7 +- .../bk/job/common/constant/ErrorCode.java | 14 ++- .../job/file_gateway/service/FileService.java | 2 + .../service/impl/FileServiceImpl.java | 33 ++++++- .../remote/FileSourceReqGenService.java | 14 ++- .../impl/FileSourceReqGenServiceImpl.java | 8 ++ .../FileSourceStatusUpdateTask.java | 17 +++- .../bk/job/file/worker/api/FileResource.java | 6 ++ .../worker/model/req/ListFileNodeReq.java | 6 ++ .../job/file/worker/api/FileResourceImpl.java | 6 ++ .../bk/job/file/worker/api/IFileResource.java | 2 + .../file/worker/api/FileResourceProxy.java | 5 + .../impl/ArtifactoryFileResourceImpl.java | 6 ++ .../service/ArtifactoryRemoteClient.java | 98 ++++++++++++------- .../worker/cos/impl/COSFileResourceImpl.java | 53 +++------- .../worker/cos/service/COSBaseService.java | 5 +- 20 files changed, 197 insertions(+), 113 deletions(-) diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties index 2f5e7da6ce..6df5eddb2d 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties @@ -132,10 +132,9 @@ 1246003=删除作业执行日志失败 ##业务错误-文件网关服务(job-file-gateway) 1260001=文件源不存在:{0} -1260002=接入点响应异常：ListBucket，详情：{0} -1260003=接入点响应异常：ListObjects，详情：{0} -1260008=接入点响应异常：DeleteBucket，详情：{0} -1260009=接入点响应异常：DeleteBucketFile，详情：{0} +1260002=接入点响应异常：ListFileNode，详情：{0} +1260003=接入点响应异常：FileAvailable，详情：{0} +1260008=接入点响应异常：ExecuteAction，详情：{0} 1260004=文件源别名已存在：{0} 1260005=无法匹配到有效接入点，请检查文件源配置 1260006=接入点响应异常：ClearTaskFiles，详情：{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties index 061da05367..0d8b812025 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties @@ -133,10 +133,9 @@ 1246003=Failed to delete execution log ## Business error (job-file-gateway) 1260001=FileSource does not exist:{0} -1260002=FileWorker response error:ListBucket, Detail:{0} -1260003=FileWorker response error:ListObjects, Detail:{0} -1260008=FileWorker response error:DeleteBucket, Detail:{0} -1260009=FileWorker response error:DeleteBucketFile, Detail:{0} +1260002=FileWorker response error:ListFileNode, Detail:{0} +1260003=FileWorker response error:FileAvailable, Detail:{0} +1260008=FileWorker response error:ExecuteAction, Detail:{0} 1260004=FileSource alias already exists:{0} 1260005=Cannot find available file worker, please check configuration of fileSource 1260006=FileWorker response error:ClearTaskFiles, Detail:{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties index 061da05367..0d8b812025 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties @@ -133,10 +133,9 @@ 1246003=Failed to delete execution log ## Business error (job-file-gateway) 1260001=FileSource does not exist:{0} -1260002=FileWorker response error:ListBucket, Detail:{0} -1260003=FileWorker response error:ListObjects, Detail:{0} -1260008=FileWorker response error:DeleteBucket, Detail:{0} -1260009=FileWorker response error:DeleteBucketFile, Detail:{0} +1260002=FileWorker response error:ListFileNode, Detail:{0} +1260003=FileWorker response error:FileAvailable, Detail:{0} +1260008=FileWorker response error:ExecuteAction, Detail:{0} 1260004=FileSource alias already exists:{0} 1260005=Cannot find available file worker, please check configuration of fileSource 1260006=FileWorker response error:ClearTaskFiles, Detail:{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties index 2f5e7da6ce..6df5eddb2d 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties @@ -132,10 +132,9 @@ 1246003=删除作业执行日志失败 ##业务错误-文件网关服务(job-file-gateway) 1260001=文件源不存在:{0} -1260002=接入点响应异常：ListBucket，详情：{0} -1260003=接入点响应异常：ListObjects，详情：{0} -1260008=接入点响应异常：DeleteBucket，详情：{0} -1260009=接入点响应异常：DeleteBucketFile，详情：{0} +1260002=接入点响应异常：ListFileNode，详情：{0} +1260003=接入点响应异常：FileAvailable，详情：{0} +1260008=接入点响应异常：ExecuteAction，详情：{0} 1260004=文件源别名已存在：{0} 1260005=无法匹配到有效接入点，请检查文件源配置 1260006=接入点响应异常：ClearTaskFiles，详情：{0} diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties index 2f5e7da6ce..6df5eddb2d 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties @@ -132,10 +132,9 @@ 1246003=删除作业执行日志失败 ##业务错误-文件网关服务(job-file-gateway) 1260001=文件源不存在:{0} -1260002=接入点响应异常：ListBucket，详情：{0} -1260003=接入点响应异常：ListObjects，详情：{0} -1260008=接入点响应异常：DeleteBucket，详情：{0} -1260009=接入点响应异常：DeleteBucketFile，详情：{0} +1260002=接入点响应异常：ListFileNode，详情：{0} +1260003=接入点响应异常：FileAvailable，详情：{0} +1260008=接入点响应异常：ExecuteAction，详情：{0} 1260004=文件源别名已存在：{0} 1260005=无法匹配到有效接入点，请检查文件源配置 1260006=接入点响应异常：ClearTaskFiles，详情：{0} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java index 454983ef24..459f53a91c 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java @@ -251,14 +251,12 @@ public class ErrorCode { // 文件网关 start // 文件源不存在:{0} public static final int FILE_SOURCE_NOT_EXIST = 1260001; - // 接入点响应异常：ListBucket，详情：{0} - public static final int FAIL_TO_REQUEST_FILE_WORKER_LIST_BUCKET = 1260002; - // 接入点响应异常：ListObjects，详情：{0} - public static final int FAIL_TO_REQUEST_FILE_WORKER_LIST_OBJECTS = 1260003; - // 接入点响应异常：DeleteBucket，详情：{0} - public static final int FAIL_TO_REQUEST_FILE_WORKER_DELETE_BUCKET = 1260008; - // 接入点响应异常：DeleteBucketFile，详情：{0} - public static final int FAIL_TO_REQUEST_FILE_WORKER_DELETE_BUCKET_FILE = 1260009; + // 接入点响应异常：ListFileNode，详情：{0} + public static final int FAIL_TO_REQUEST_FILE_WORKER_LIST_FILE_NODE = 1260002; + // 接入点响应异常：FileAvailable，详情：{0} + public static final int FAIL_TO_REQUEST_FILE_WORKER_FILE_AVAILABLE = 1260003; + // 接入点响应异常：ExecuteAction，详情：{0} + public static final int FAIL_TO_REQUEST_FILE_WORKER_EXECUTE_ACTION = 1260008; // 文件源别名已存在：{0} public static final int FILE_SOURCE_ALIAS_ALREADY_EXISTS = 1260004; diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileService.java index e8dc9b31fb..f6de456397 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileService.java @@ -29,6 +29,8 @@ public interface FileService { + boolean isFileAvailable(String username, Long appId, Integer fileSourceId); + FileNodesVO listFileNode(String username, Long appId, Integer fileSourceId, String path, String name, Integer start, Integer pageSize); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileServiceImpl.java index 063c62defc..f58212d09c 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileServiceImpl.java @@ -69,6 +69,33 @@ private FileWorkerDTO getFileWorker(Long appId, FileSourceDTO fileSourceDTO) { return dispatchService.findBestFileWorker(fileSourceDTO); } + @Override + public boolean isFileAvailable(String username, Long appId, Integer fileSourceId) { + FileSourceDTO fileSourceDTO = fileSourceService.getFileSourceById(appId, fileSourceId); + FileWorkerDTO fileWorkerDTO = getFileWorker(appId, fileSourceDTO); + if (fileWorkerDTO == null) { + throw new ServiceException(ErrorCode.CAN_NOT_FIND_AVAILABLE_FILE_WORKER); + } + log.info("choose file worker:" + fileWorkerDTO); + // 访问文件Worker接口，拿到available状态信息 + HttpReq fileAvailableReq = fileSourceReqGenService.genFileAvailableReq(appId, fileWorkerDTO, fileSourceDTO); + String respStr = null; + log.info(String.format("url=%s,body=%s,headers=%s", fileAvailableReq.getUrl(), fileAvailableReq.getBody(), + JsonUtils.toJson(fileAvailableReq.getHeaders()))); + try { + respStr = fileWorkerHttpHelper.post(fileAvailableReq.getUrl(), fileAvailableReq.getBody(), + fileAvailableReq.getHeaders()); + log.info(String.format("respStr=%s", respStr)); + ServiceResponse resp = JsonUtils.fromJson(respStr, + new TypeReference>() { + }); + return resp.getData(); + } catch (Exception e) { + log.error("Fail to request remote worker:", e); + return false; + } + } + @Override public FileNodesVO listFileNode(String username, Long appId, Integer fileSourceId, String path, String name, Integer start, Integer pageSize) { @@ -91,7 +118,7 @@ public FileNodesVO listFileNode(String username, Long appId, Integer fileSourceI listFileNodeReq.getHeaders()); } catch (Exception e) { log.error("Fail to request remote worker:", e); - throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_LIST_BUCKET, new String[]{e.getMessage()}); + throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_LIST_FILE_NODE, new String[]{e.getMessage()}); } log.info(String.format("respStr=%s", respStr)); FileNodesDTO fileNodesDTO = parseFileNodesDTO(respStr); @@ -127,7 +154,7 @@ public Boolean executeAction(String username, Long appId, Integer fileSourceId, throw (ServiceException) e; } else { log.error("Fail to request remote worker:", e); - throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_DELETE_BUCKET, + throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_EXECUTE_ACTION, new String[]{e.getMessage()}); } } @@ -140,7 +167,7 @@ private FileNodesDTO parseFileNodesDTO(String respStr) { }); } catch (Exception e) { log.error("Fail to parse bucket from response={}", respStr, e); - throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_LIST_BUCKET, e.getMessage()); + throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_LIST_FILE_NODE, e.getMessage()); } if (resp.isSuccess()) { return resp.getData(); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceReqGenService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceReqGenService.java index 35131fa337..85988d99a7 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceReqGenService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceReqGenService.java @@ -34,15 +34,21 @@ * 远程文件源请求生成 */ public interface FileSourceReqGenService { + + /** + * 生成FileAvailable请求 + */ + HttpReq genFileAvailableReq(Long appId, FileWorkerDTO fileWorkerDTO, FileSourceDTO fileSourceDTO); + /** - * 生成ListBucket请求 - * - * @param fileWorkerDTO - * @return + * 生成ListFileNode请求 */ HttpReq genListFileNodeReq(Long appId, String path, String name, Integer start, Integer pageSize, FileWorkerDTO fileWorkerDTO, FileSourceDTO fileSourceDTO); + /** + * 生成ExecuteAction请求 + */ HttpReq genExecuteActionReq(Long appId, String actionCode, Map actionParams, FileWorkerDTO fileWorkerDTO, FileSourceDTO fileSourceDTO); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/FileSourceReqGenServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/FileSourceReqGenServiceImpl.java index ccf7837637..dc47f5b09e 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/FileSourceReqGenServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/FileSourceReqGenServiceImpl.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.file_gateway.service.remote.impl; import com.tencent.bk.job.common.model.http.HttpReq; +import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq; import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq; import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; @@ -45,6 +46,13 @@ public FileSourceReqGenServiceImpl(CredentialService credentialService) { super(credentialService); } + @Override + public HttpReq genFileAvailableReq(Long appId, FileWorkerDTO fileWorkerDTO, FileSourceDTO fileSourceDTO) { + BaseReq req = new BaseReq(); + String url = fillBaseReqGetUrl(req, appId, fileWorkerDTO, fileSourceDTO, "/file/available"); + return genRemoteFileReq(url, req); + } + @Override public HttpReq genListFileNodeReq(Long appId, String path, String name, Integer start, Integer pageSize, FileWorkerDTO fileWorkerDTO, FileSourceDTO fileSourceDTO) { diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/filesource/FileSourceStatusUpdateTask.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/filesource/FileSourceStatusUpdateTask.java index 2deab3bcfe..ecbc9419c6 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/filesource/FileSourceStatusUpdateTask.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/filesource/FileSourceStatusUpdateTask.java @@ -57,7 +57,13 @@ public void run() { int start = 0; int pageSize = 20; do { - fileSourceDTOList = fileSourceService.listWorkTableFileSource(null, null, null, start, pageSize); + fileSourceDTOList = fileSourceService.listWorkTableFileSource( + null, + null, + null, + start, + pageSize + ); for (FileSourceDTO fileSourceDTO : fileSourceDTOList) { FileWorkerDTO fileWorkerDTO = dispatchService.findBestFileWorker(fileSourceDTO); int status; @@ -70,9 +76,12 @@ public void run() { } else { // 通过Worker调用listFileNode接口，OK的才算正常 try { - fileService.listFileNode(fileSourceDTO.getCreator(), fileSourceDTO.getAppId(), - fileSourceDTO.getId(), null, null, 0, 1); - status = 1; + if (fileService.isFileAvailable(fileSourceDTO.getCreator(), fileSourceDTO.getAppId(), + fileSourceDTO.getId())) { + status = 1; + } else { + status = 0; + } } catch (Throwable t) { status = 0; } diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResource.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResource.java index 8f15d9e0c7..bae8a170eb 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResource.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResource.java @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.ServiceResponse; +import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq; import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq; import com.tencent.bk.job.file_gateway.model.resp.common.FileNodesDTO; @@ -44,6 +45,11 @@ @InternalAPI public interface FileResource { + @ApiOperation(value = "测试文件源是否可用", produces = "application/json") + @PostMapping("/available") + ServiceResponse isFileAvailable( + @ApiParam(value = "文件源是否可用", required = true) @RequestBody BaseReq req); + @ApiOperation(value = "获取文件源/FileNode下的子FileNode列表", produces = "application/json") @PostMapping("/listFileNode") ServiceResponse listFileNode( diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ListFileNodeReq.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ListFileNodeReq.java index 444d4b3e4f..3257cce07d 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ListFileNodeReq.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ListFileNodeReq.java @@ -42,4 +42,10 @@ public class ListFileNodeReq extends BaseReq { String name; Integer start; Integer pageSize; + + public ListFileNodeReq(BaseReq req) { + this.setCredential(req.getCredential()); + this.setFileSourceTypeCode(req.getFileSourceTypeCode()); + this.setFileSourceInfoMap(req.getFileSourceInfoMap()); + } } diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceImpl.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceImpl.java index 84b1a21b60..272e0d8f4c 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceImpl.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceImpl.java @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.model.ServiceResponse; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq; import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq; import com.tencent.bk.job.file_gateway.model.resp.common.FileNodesDTO; @@ -45,6 +46,11 @@ protected FileResourceImpl(IFileResource fileResource) { this.fileResource = fileResource; } + @Override + public ServiceResponse isFileAvailable(BaseReq req) { + return fileResource.isFileAvailable(req); + } + @Override public ServiceResponse listFileNode(ListFileNodeReq req) { log.info("req={}", JsonUtils.toJson(req)); diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/IFileResource.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/IFileResource.java index 3afcc64f01..ee19270f40 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/IFileResource.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/IFileResource.java @@ -35,6 +35,8 @@ public interface IFileResource { RemoteClient getRemoteClient(BaseReq req); + ServiceResponse isFileAvailable(BaseReq req); + ServiceResponse listFileNode(ListFileNodeReq req); ServiceResponse executeAction(ExecuteActionReq req); diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceProxy.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceProxy.java index ca59790ac4..2996956412 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceProxy.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceProxy.java @@ -69,6 +69,11 @@ public RemoteClient getRemoteClient(BaseReq req) { return chooseFileResource(req).getRemoteClient(req); } + @Override + public ServiceResponse isFileAvailable(BaseReq req) { + return chooseFileResource(req).isFileAvailable(req); + } + @Override public ServiceResponse listFileNode(ListFileNodeReq req) { return chooseFileResource(req).listFileNode(req); diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/impl/ArtifactoryFileResourceImpl.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/impl/ArtifactoryFileResourceImpl.java index b3c66675b6..f8225a55ab 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/impl/ArtifactoryFileResourceImpl.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/impl/ArtifactoryFileResourceImpl.java @@ -75,6 +75,12 @@ public RemoteClient getRemoteClient(BaseReq req) { return baseService.getArtifactoryClientFromBaseReq(req); } + @Override + public ServiceResponse isFileAvailable(BaseReq req) { + ArtifactoryRemoteClient client = baseService.getArtifactoryClientFromBaseReq(req); + return ServiceResponse.buildSuccessResp(client.isAvailable()); + } + private String parseParentNodeTypeByPath(String path) { if (StringUtils.isBlank(path)) return ArtifactoryNodeTypeEnum.FILE_SOURCE.name(); path = StringUtil.removePrefix(path, "/"); diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java index 31723c8f77..c513fe83b1 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java @@ -34,8 +34,19 @@ import com.tencent.bk.job.common.util.http.LongRetryableHttpHelper; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.file.worker.artifactory.consts.ArtifactoryInterfaceConsts; -import com.tencent.bk.job.file.worker.artifactory.model.dto.*; -import com.tencent.bk.job.file.worker.artifactory.model.req.*; +import com.tencent.bk.job.file.worker.artifactory.model.dto.ArtifactoryResp; +import com.tencent.bk.job.file.worker.artifactory.model.dto.NodeDTO; +import com.tencent.bk.job.file.worker.artifactory.model.dto.PageData; +import com.tencent.bk.job.file.worker.artifactory.model.dto.ProjectDTO; +import com.tencent.bk.job.file.worker.artifactory.model.dto.RepoDTO; +import com.tencent.bk.job.file.worker.artifactory.model.req.ArtifactoryReq; +import com.tencent.bk.job.file.worker.artifactory.model.req.DeleteNodeReq; +import com.tencent.bk.job.file.worker.artifactory.model.req.DeleteRepoReq; +import com.tencent.bk.job.file.worker.artifactory.model.req.DownloadGenericFileReq; +import com.tencent.bk.job.file.worker.artifactory.model.req.ListNodePageReq; +import com.tencent.bk.job.file.worker.artifactory.model.req.ListProjectReq; +import com.tencent.bk.job.file.worker.artifactory.model.req.ListRepoPageReq; +import com.tencent.bk.job.file.worker.artifactory.model.req.QueryNodeDetailReq; import com.tencent.bk.job.file.worker.cos.service.RemoteClient; import com.tencent.bk.job.file.worker.model.FileMetaData; import io.micrometer.core.instrument.MeterRegistry; @@ -52,11 +63,13 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; @Slf4j public class ArtifactoryRemoteClient implements RemoteClient { + public static final String URL_ACTUATOR_INFO = "/repository/actuator/info"; public static final String URL_LIST_PROJECT = "/repository/api/project/list"; public static final String URL_LIST_REPO_PAGE = "/repository/api/repo/page/{projectId}/{pageNumber}/{pageSize}"; public static final String URL_LIST_NODE_PAGE = "/repository/api/node/page/{projectId}/{repoName}/{fullPath}"; @@ -154,34 +167,36 @@ private R getArtifactoryRespByReq( log.debug("success|method={}|url={}|reqStr={}|respStr={}", method, url, reqStr, respStr); } R result = JsonUtils.fromJson(respStr, typeReference); - ArtifactoryResp artifactoryResp = (ArtifactoryResp) result; - if (artifactoryResp == null) { - log.error("fail:artifactoryResp is null after parse|method={}|url={}|reqStr={}|respStr={}", method, - url, reqStr, respStr); - status = "error"; - throw new ServiceException(ErrorCode.ARTIFACTORY_API_DATA_ERROR, "artifactoryResp is null after parse"); - } else if (artifactoryResp.getCode() != ArtifactoryInterfaceConsts.RESULT_CODE_OK) { - log.error( - "fail:artifactoryResp code!={}|artifactoryResp.requestId={}|artifactoryResp" + - ".code={}|artifactoryResp.message={}|method={}|url={}|reqStr={}|respStr={}" - , ArtifactoryInterfaceConsts.RESULT_CODE_OK - , artifactoryResp.getTraceId() - , artifactoryResp.getCode() - , artifactoryResp.getMessage() - , method, url, reqStr, respStr - ); - status = "error"; - throw new ServiceException(ErrorCode.ARTIFACTORY_API_DATA_ERROR, "artifactoryResp code!=0"); - } - if (artifactoryResp.getData() == null) { - log.warn( - "warn:artifactoryResp.getData() == null|artifactoryResp.requestId={}|artifactoryResp" + - ".code={}|artifactoryResp.message={}|method={}|url={}|reqStr={}|respStr={}" - , artifactoryResp.getTraceId() - , artifactoryResp.getCode() - , artifactoryResp.getMessage() - , method, url, reqStr, respStr - ); + if (result instanceof ArtifactoryResp) { + ArtifactoryResp artifactoryResp = (ArtifactoryResp) result; + if (artifactoryResp == null) { + log.error("fail:artifactoryResp is null after parse|method={}|url={}|reqStr={}|respStr={}", method, + url, reqStr, respStr); + status = "error"; + throw new ServiceException(ErrorCode.ARTIFACTORY_API_DATA_ERROR, "artifactoryResp is null after parse"); + } else if (artifactoryResp.getCode() != ArtifactoryInterfaceConsts.RESULT_CODE_OK) { + log.error( + "fail:artifactoryResp code!={}|artifactoryResp.requestId={}|artifactoryResp" + + ".code={}|artifactoryResp.message={}|method={}|url={}|reqStr={}|respStr={}" + , ArtifactoryInterfaceConsts.RESULT_CODE_OK + , artifactoryResp.getTraceId() + , artifactoryResp.getCode() + , artifactoryResp.getMessage() + , method, url, reqStr, respStr + ); + status = "error"; + throw new ServiceException(ErrorCode.ARTIFACTORY_API_DATA_ERROR, "artifactoryResp code!=0"); + } + if (artifactoryResp.getData() == null) { + log.warn( + "warn:artifactoryResp.getData() == null|artifactoryResp.requestId={}|artifactoryResp" + + ".code={}|artifactoryResp.message={}|method={}|url={}|reqStr={}|respStr={}" + , artifactoryResp.getTraceId() + , artifactoryResp.getCode() + , artifactoryResp.getMessage() + , method, url, reqStr, respStr + ); + } } status = "ok"; return result; @@ -198,10 +213,21 @@ private R getArtifactoryRespByReq( } } + public boolean isAvailable() { + try { + getArtifactoryRespByReq(HttpGet.METHOD_NAME, URL_ACTUATOR_INFO, + new ArtifactoryReq(), new TypeReference>() { + }, httpHelper); + return true; + } catch (Throwable t) { + return false; + } + } + public List listProject() { ArtifactoryResp> resp = getArtifactoryRespByReq(HttpGet.METHOD_NAME, URL_LIST_PROJECT, new ListProjectReq(), new TypeReference>>() { - }, httpHelper); + }, httpHelper); return resp.getData(); } @@ -213,7 +239,7 @@ public PageData listRepo(String projectId, String name, int pageNumber, req.setPageSize(pageSize); ArtifactoryResp> resp = getArtifactoryRespByReq(HttpGet.METHOD_NAME, URL_LIST_REPO_PAGE, req, new TypeReference>>() { - }, httpHelper); + }, httpHelper); return resp.getData(); } @@ -227,7 +253,7 @@ public PageData listNode(String projectId, String repoName, String full req.setPageSize(pageSize); ArtifactoryResp> resp = getArtifactoryRespByReq(HttpGet.METHOD_NAME, URL_LIST_NODE_PAGE, req, new TypeReference>>() { - }, httpHelper); + }, httpHelper); return resp.getData(); } @@ -238,7 +264,7 @@ public NodeDTO queryNodeDetail(String projectId, String repoName, String fullPat req.setFullPath(fullPath); ArtifactoryResp resp = getArtifactoryRespByReq(HttpGet.METHOD_NAME, URL_QUERY_NODE_DETAIL, req, new TypeReference>() { - }, httpHelper); + }, httpHelper); return resp.getData(); } @@ -253,7 +279,7 @@ public Boolean deleteRepo(String projectId, String repoName, Boolean forced) { req.setForced(forced); ArtifactoryResp resp = getArtifactoryRespByReq(HttpDelete.METHOD_NAME, URL_DELETE_REPO, req, new TypeReference>() { - }, httpHelper); + }, httpHelper); return resp.getCode() == 0; } @@ -264,7 +290,7 @@ public Boolean deleteNode(String projectId, String repoName, String fullPath) { req.setFullPath(fullPath); ArtifactoryResp resp = getArtifactoryRespByReq(HttpDelete.METHOD_NAME, URL_DELETE_NODE, req, new TypeReference>() { - }, httpHelper); + }, httpHelper); return resp.getCode() == 0; } diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/impl/COSFileResourceImpl.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/impl/COSFileResourceImpl.java index 84a5afa26b..0446d8c032 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/impl/COSFileResourceImpl.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/impl/COSFileResourceImpl.java @@ -24,7 +24,6 @@ package com.tencent.bk.job.file.worker.cos.impl; -import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.ServiceResponse; @@ -32,7 +31,6 @@ import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.file.PathUtil; -import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.file.worker.api.IFileResource; import com.tencent.bk.job.file.worker.cos.JobTencentInnerCOSClient; import com.tencent.bk.job.file.worker.cos.consts.COSActionCodeEnum; @@ -48,7 +46,6 @@ import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq; import com.tencent.bk.job.file_gateway.model.resp.common.FileNodesDTO; import com.tencent.bk.job.file_gateway.model.resp.common.FileTreeNodeDef; -import com.tencent.bk.job.file_gateway.model.resp.common.FileVO; import com.tencent.cos.model.Bucket; import com.tencent.cos.model.COSObjectSummary; import io.micrometer.core.instrument.util.StringUtils; @@ -207,6 +204,21 @@ public RemoteClient getRemoteClient(BaseReq req) { return new COSRemoteClient(jobTencentInnerCOSClient); } + @Override + public ServiceResponse isFileAvailable(BaseReq req) { + try { + ListFileNodeReq listFileNodeReq = new ListFileNodeReq(req); + listFileNodeReq.setPath(""); + listFileNodeReq.setName(""); + listFileNodeReq.setStart(0); + listFileNodeReq.setPageSize(1); + listBucket(listFileNodeReq); + return ServiceResponse.buildSuccessResp(true); + } catch (Throwable t) { + return ServiceResponse.buildSuccessResp(false); + } + } + private String getTypeFromFileName(String fileName) { if (StringUtils.isBlank(fileName)) { return "UNKNOWN"; @@ -233,39 +245,6 @@ private Boolean isDir(String fileName) { return fileName.endsWith("/"); } - private List parseBucketFileList(String bucketName, String path, String respStr) { - ServiceResponse> resp = null; - if (path == null) { - path = ""; - } - try { - resp = JsonUtils.fromJson(respStr, new TypeReference>>() { - }); - } catch (Exception e) { - log.error("Fail to parse bucket file from response={}", respStr, e); - throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_LIST_OBJECTS, resp.getErrorMsg()); - } - if (resp.isSuccess()) { - List fileVOList = new ArrayList<>(); - List fileDTOList = resp.getData(); - for (FileDTO fileDTO : fileDTOList) { - FileVO fileVO = new FileVO(); - fileVO.setName(fileDTO.getKey()); - fileVO.setCompletePath(bucketName + "/" + path + fileDTO.getKey()); - fileVO.setSize(fileDTO.getSize()); - fileVO.setDir(isDir(fileDTO.getKey())); - fileVO.setType(getTypeFromFileName(fileDTO.getKey())); - fileVO.setDownloadUrl(fileDTO.getDownloadUrl()); - fileVO.setLastModifyTime(fileDTO.getLastModified()); - fileVOList.add(fileVO); - } - return fileVOList; - } else { - log.error("get failed bucket file response={}", respStr); - throw new ServiceException(resp.getCode(), resp.getErrorMsg()); - } - } - private int getSlashNum(String str) { if (StringUtils.isBlank(str)) return 0; int count = 0; @@ -342,7 +321,7 @@ private void fillFileFileNodesDTO(FileNodesDTO fileNodesDTO, ListFileNodeReq req Map map = new HashMap<>(); String fileName = fileDTO.getKey(); map.put("name", fileName); - map.put("type", "文本文件"); + map.put("type", getTypeFromFileName(fileName)); map.put("updateTime", DateUtils.formatUnixTimestamp(fileDTO.getLastModified(), ChronoUnit.MILLIS)); map.put("completePath", PathUtil.joinFilePath(req.getPath(), fileName)); map.put("dir", isDir(fileName)); diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSBaseService.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSBaseService.java index 812747adc2..53059afced 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSBaseService.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSBaseService.java @@ -27,10 +27,12 @@ import com.tencent.bk.job.file.worker.cos.JobTencentInnerCOSClient; import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.ticket.model.credential.CommonCredential; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.util.Map; +@Slf4j @Service public class COSBaseService { @@ -39,8 +41,9 @@ public JobTencentInnerCOSClient getCOSClientFromBaseReq(BaseReq req) { // appId CommonCredential credential = req.getCredential(); Map fileSourceInfoMap = req.getFileSourceInfoMap(); + log.debug("req={}", req); return new JobTencentInnerCOSClient(credential.getAccessKey(), credential.getSecretKey(), - getEndPointDomain(req), fileSourceInfoMap.get("appId").toString()); + getEndPointDomain(req), fileSourceInfoMap.get("app_id").toString()); } public String getEndPointDomain(BaseReq req) {