feature: 提供文件源管理、凭据管理相关ESB API TencentBlueKing#92

hLinx · Sep 16, 2021 · 17ea023 · 17ea023
1 parent 3b4b13c
commit 17ea023
Show file tree

Hide file tree

Showing 20 changed files with 197 additions and 113 deletions.
diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties
@@ -133,10 +133,9 @@
 1246003=删除作业执行日志失败
 ##业务错误-文件网关服务(job-file-gateway)
 1260001=文件源不存在:{0}
-1260002=接入点响应异常：ListBucket，详情：{0}
-1260003=接入点响应异常：ListObjects，详情：{0}
-1260008=接入点响应异常：DeleteBucket，详情：{0}
-1260009=接入点响应异常：DeleteBucketFile，详情：{0}
+1260002=接入点响应异常：ListFileNode，详情：{0}
+1260003=接入点响应异常：FileAvailable，详情：{0}
+1260008=接入点响应异常：ExecuteAction，详情：{0}
 1260004=文件源别名已存在：{0}
 1260005=无法匹配到有效接入点，请检查文件源配置
 1260006=接入点响应异常：ClearTaskFiles，详情：{0}

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties
@@ -134,10 +134,9 @@
 1246003=Failed to delete execution log
 ## Business error (job-file-gateway)
 1260001=FileSource does not exist:{0}
-1260002=FileWorker response error:ListBucket, Detail:{0}
-1260003=FileWorker response error:ListObjects, Detail:{0}
-1260008=FileWorker response error:DeleteBucket, Detail:{0}
-1260009=FileWorker response error:DeleteBucketFile, Detail:{0}
+1260002=FileWorker response error:ListFileNode, Detail:{0}
+1260003=FileWorker response error:FileAvailable, Detail:{0}
+1260008=FileWorker response error:ExecuteAction, Detail:{0}
 1260004=FileSource alias already exists:{0}
 1260005=Cannot find available file worker, please check configuration of fileSource
 1260006=FileWorker response error:ClearTaskFiles, Detail:{0}

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties
@@ -134,10 +134,9 @@
 1246003=Failed to delete execution log
 ## Business error (job-file-gateway)
 1260001=FileSource does not exist:{0}
-1260002=FileWorker response error:ListBucket, Detail:{0}
-1260003=FileWorker response error:ListObjects, Detail:{0}
-1260008=FileWorker response error:DeleteBucket, Detail:{0}
-1260009=FileWorker response error:DeleteBucketFile, Detail:{0}
+1260002=FileWorker response error:ListFileNode, Detail:{0}
+1260003=FileWorker response error:FileAvailable, Detail:{0}
+1260008=FileWorker response error:ExecuteAction, Detail:{0}
 1260004=FileSource alias already exists:{0}
 1260005=Cannot find available file worker, please check configuration of fileSource
 1260006=FileWorker response error:ClearTaskFiles, Detail:{0}

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties
@@ -133,10 +133,9 @@
 1246003=删除作业执行日志失败
 ##业务错误-文件网关服务(job-file-gateway)
 1260001=文件源不存在:{0}
-1260002=接入点响应异常：ListBucket，详情：{0}
-1260003=接入点响应异常：ListObjects，详情：{0}
-1260008=接入点响应异常：DeleteBucket，详情：{0}
-1260009=接入点响应异常：DeleteBucketFile，详情：{0}
+1260002=接入点响应异常：ListFileNode，详情：{0}
+1260003=接入点响应异常：FileAvailable，详情：{0}
+1260008=接入点响应异常：ExecuteAction，详情：{0}
 1260004=文件源别名已存在：{0}
 1260005=无法匹配到有效接入点，请检查文件源配置
 1260006=接入点响应异常：ClearTaskFiles，详情：{0}

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties
@@ -133,10 +133,9 @@
 1246003=删除作业执行日志失败
 ##业务错误-文件网关服务(job-file-gateway)
 1260001=文件源不存在:{0}
-1260002=接入点响应异常：ListBucket，详情：{0}
-1260003=接入点响应异常：ListObjects，详情：{0}
-1260008=接入点响应异常：DeleteBucket，详情：{0}
-1260009=接入点响应异常：DeleteBucketFile，详情：{0}
+1260002=接入点响应异常：ListFileNode，详情：{0}
+1260003=接入点响应异常：FileAvailable，详情：{0}
+1260008=接入点响应异常：ExecuteAction，详情：{0}
 1260004=文件源别名已存在：{0}
 1260005=无法匹配到有效接入点，请检查文件源配置
 1260006=接入点响应异常：ClearTaskFiles，详情：{0}

diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java
@@ -252,14 +252,12 @@ public class ErrorCode {
     // 文件网关 start
     // 文件源不存在:{0}
     public static final int FILE_SOURCE_NOT_EXIST = 1260001;
-    // 接入点响应异常：ListBucket，详情：{0}
-    public static final int FAIL_TO_REQUEST_FILE_WORKER_LIST_BUCKET = 1260002;
-    // 接入点响应异常：ListObjects，详情：{0}
-    public static final int FAIL_TO_REQUEST_FILE_WORKER_LIST_OBJECTS = 1260003;
-    // 接入点响应异常：DeleteBucket，详情：{0}
-    public static final int FAIL_TO_REQUEST_FILE_WORKER_DELETE_BUCKET = 1260008;
-    // 接入点响应异常：DeleteBucketFile，详情：{0}
-    public static final int FAIL_TO_REQUEST_FILE_WORKER_DELETE_BUCKET_FILE = 1260009;
+    // 接入点响应异常：ListFileNode，详情：{0}
+    public static final int FAIL_TO_REQUEST_FILE_WORKER_LIST_FILE_NODE = 1260002;
+    // 接入点响应异常：FileAvailable，详情：{0}
+    public static final int FAIL_TO_REQUEST_FILE_WORKER_FILE_AVAILABLE = 1260003;
+    // 接入点响应异常：ExecuteAction，详情：{0}
+    public static final int FAIL_TO_REQUEST_FILE_WORKER_EXECUTE_ACTION = 1260008;
 
     // 文件源别名已存在：{0}
     public static final int FILE_SOURCE_ALIAS_ALREADY_EXISTS = 1260004;

diff --git a/...e-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileService.java b/...e-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileService.java
@@ -29,6 +29,8 @@
 
 public interface FileService {
 
+    boolean isFileAvailable(String username, Long appId, Integer fileSourceId);
+
     FileNodesVO listFileNode(String username, Long appId, Integer fileSourceId, String path, String name,
                              Integer start, Integer pageSize);
 

diff --git a/...e-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileServiceImpl.java b/...e-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileServiceImpl.java
@@ -69,6 +69,33 @@ private FileWorkerDTO getFileWorker(Long appId, FileSourceDTO fileSourceDTO) {
         return dispatchService.findBestFileWorker(fileSourceDTO);
     }
 
+    @Override
+    public boolean isFileAvailable(String username, Long appId, Integer fileSourceId) {
+        FileSourceDTO fileSourceDTO = fileSourceService.getFileSourceById(appId, fileSourceId);
+        FileWorkerDTO fileWorkerDTO = getFileWorker(appId, fileSourceDTO);
+        if (fileWorkerDTO == null) {
+            throw new ServiceException(ErrorCode.CAN_NOT_FIND_AVAILABLE_FILE_WORKER);
+        }
+        log.info("choose file worker:" + fileWorkerDTO);
+        // 访问文件Worker接口，拿到available状态信息
+        HttpReq fileAvailableReq = fileSourceReqGenService.genFileAvailableReq(appId, fileWorkerDTO, fileSourceDTO);
+        String respStr = null;
+        log.info(String.format("url=%s,body=%s,headers=%s", fileAvailableReq.getUrl(), fileAvailableReq.getBody(),
+            JsonUtils.toJson(fileAvailableReq.getHeaders())));
+        try {
+            respStr = fileWorkerHttpHelper.post(fileAvailableReq.getUrl(), fileAvailableReq.getBody(),
+                fileAvailableReq.getHeaders());
+            log.info(String.format("respStr=%s", respStr));
+            ServiceResponse<Boolean> resp = JsonUtils.fromJson(respStr,
+                new TypeReference<ServiceResponse<Boolean>>() {
+                });
+            return resp.getData();
+        } catch (Exception e) {
+            log.error("Fail to request remote worker:", e);
+            return false;
+        }
+    }
+
     @Override
     public FileNodesVO listFileNode(String username, Long appId, Integer fileSourceId, String path, String name,
                                     Integer start, Integer pageSize) {
@@ -91,7 +118,7 @@ public FileNodesVO listFileNode(String username, Long appId, Integer fileSourceI
                 listFileNodeReq.getHeaders());
         } catch (Exception e) {
             log.error("Fail to request remote worker:", e);
-            throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_LIST_BUCKET, new String[]{e.getMessage()});
+            throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_LIST_FILE_NODE, new String[]{e.getMessage()});
         }
         log.info(String.format("respStr=%s", respStr));
         FileNodesDTO fileNodesDTO = parseFileNodesDTO(respStr);
@@ -127,7 +154,7 @@ public Boolean executeAction(String username, Long appId, Integer fileSourceId,
                 throw (ServiceException) e;
             } else {
                 log.error("Fail to request remote worker:", e);
-                throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_DELETE_BUCKET,
+                throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_EXECUTE_ACTION,
                     new String[]{e.getMessage()});
             }
         }
@@ -140,7 +167,7 @@ private FileNodesDTO parseFileNodesDTO(String respStr) {
             });
         } catch (Exception e) {
             log.error("Fail to parse bucket from response={}", respStr, e);
-            throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_LIST_BUCKET, e.getMessage());
+            throw new ServiceException(ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_LIST_FILE_NODE, e.getMessage());
         }
         if (resp.isSuccess()) {
             return resp.getData();

diff --git a/...src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceReqGenService.java b/...src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceReqGenService.java
@@ -34,15 +34,21 @@
  * 远程文件源请求生成
  */
 public interface FileSourceReqGenService {
+
+    /**
+     * 生成FileAvailable请求
+     */
+    HttpReq genFileAvailableReq(Long appId, FileWorkerDTO fileWorkerDTO, FileSourceDTO fileSourceDTO);
+
     /**
-     * 生成ListBucket请求
-     *
-     * @param fileWorkerDTO
-     * @return
+     * 生成ListFileNode请求
      */
     HttpReq genListFileNodeReq(Long appId, String path, String name, Integer start, Integer pageSize,
                                FileWorkerDTO fileWorkerDTO, FileSourceDTO fileSourceDTO);
 
+    /**
+     * 生成ExecuteAction请求
+     */
     HttpReq genExecuteActionReq(Long appId, String actionCode, Map<String, Object> actionParams,
                                 FileWorkerDTO fileWorkerDTO, FileSourceDTO fileSourceDTO);
 

diff --git a/...java/com/tencent/bk/job/file_gateway/service/remote/impl/FileSourceReqGenServiceImpl.java b/...java/com/tencent/bk/job/file_gateway/service/remote/impl/FileSourceReqGenServiceImpl.java
@@ -25,6 +25,7 @@
 package com.tencent.bk.job.file_gateway.service.remote.impl;
 
 import com.tencent.bk.job.common.model.http.HttpReq;
+import com.tencent.bk.job.file.worker.model.req.BaseReq;
 import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq;
 import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq;
 import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO;
@@ -45,6 +46,13 @@ public FileSourceReqGenServiceImpl(CredentialService credentialService) {
         super(credentialService);
     }
 
+    @Override
+    public HttpReq genFileAvailableReq(Long appId, FileWorkerDTO fileWorkerDTO, FileSourceDTO fileSourceDTO) {
+        BaseReq req = new BaseReq();
+        String url = fillBaseReqGetUrl(req, appId, fileWorkerDTO, fileSourceDTO, "/file/available");
+        return genRemoteFileReq(url, req);
+    }
+
     @Override
     public HttpReq genListFileNodeReq(Long appId, String path, String name, Integer start, Integer pageSize,
                                       FileWorkerDTO fileWorkerDTO, FileSourceDTO fileSourceDTO) {

diff --git a/...main/java/com/tencent/bk/job/file_gateway/task/filesource/FileSourceStatusUpdateTask.java b/...main/java/com/tencent/bk/job/file_gateway/task/filesource/FileSourceStatusUpdateTask.java
@@ -57,7 +57,13 @@ public void run() {
         int start = 0;
         int pageSize = 20;
         do {
-            fileSourceDTOList = fileSourceService.listWorkTableFileSource(null, null, null, start, pageSize);
+            fileSourceDTOList = fileSourceService.listWorkTableFileSource(
+                null,
+                null,
+                null,
+                start,
+                pageSize
+            );
             for (FileSourceDTO fileSourceDTO : fileSourceDTOList) {
                 FileWorkerDTO fileWorkerDTO = dispatchService.findBestFileWorker(fileSourceDTO);
                 int status;
@@ -70,9 +76,12 @@ public void run() {
                     } else {
                         // 通过Worker调用listFileNode接口，OK的才算正常
                         try {
-                            fileService.listFileNode(fileSourceDTO.getCreator(), fileSourceDTO.getAppId(),
-                                fileSourceDTO.getId(), null, null, 0, 1);
-                            status = 1;
+                            if (fileService.isFileAvailable(fileSourceDTO.getCreator(), fileSourceDTO.getAppId(),
+                                fileSourceDTO.getId())) {
+                                status = 1;
+                            } else {
+                                status = 0;
+                            }
                         } catch (Throwable t) {
                             status = 0;
                         }

diff --git a/...pi-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResource.java b/...pi-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResource.java
@@ -26,6 +26,7 @@
 
 import com.tencent.bk.job.common.annotation.InternalAPI;
 import com.tencent.bk.job.common.model.ServiceResponse;
+import com.tencent.bk.job.file.worker.model.req.BaseReq;
 import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq;
 import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq;
 import com.tencent.bk.job.file_gateway.model.resp.common.FileNodesDTO;
@@ -44,6 +45,11 @@
 @InternalAPI
 public interface FileResource {
 
+    @ApiOperation(value = "测试文件源是否可用", produces = "application/json")
+    @PostMapping("/available")
+    ServiceResponse<Boolean> isFileAvailable(
+        @ApiParam(value = "文件源是否可用", required = true) @RequestBody BaseReq req);
+
     @ApiOperation(value = "获取文件源/FileNode下的子FileNode列表", produces = "application/json")
     @PostMapping("/listFileNode")
     ServiceResponse<FileNodesDTO> listFileNode(

diff --git a/...le-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ListFileNodeReq.java b/...le-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ListFileNodeReq.java
@@ -42,4 +42,10 @@ public class ListFileNodeReq extends BaseReq {
     String name;
     Integer start;
     Integer pageSize;
+
+    public ListFileNodeReq(BaseReq req) {
+        this.setCredential(req.getCredential());
+        this.setFileSourceTypeCode(req.getFileSourceTypeCode());
+        this.setFileSourceInfoMap(req.getFileSourceInfoMap());
+    }
 }
diff --git a/...ob-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceImpl.java b/...ob-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceImpl.java
@@ -26,6 +26,7 @@
 
 import com.tencent.bk.job.common.model.ServiceResponse;
 import com.tencent.bk.job.common.util.json.JsonUtils;
+import com.tencent.bk.job.file.worker.model.req.BaseReq;
 import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq;
 import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq;
 import com.tencent.bk.job.file_gateway.model.resp.common.FileNodesDTO;
@@ -45,6 +46,11 @@ protected FileResourceImpl(IFileResource fileResource) {
         this.fileResource = fileResource;
     }
 
+    @Override
+    public ServiceResponse<Boolean> isFileAvailable(BaseReq req) {
+        return fileResource.isFileAvailable(req);
+    }
+
     @Override
     public ServiceResponse<FileNodesDTO> listFileNode(ListFileNodeReq req) {
         log.info("req={}", JsonUtils.toJson(req));

diff --git a/...e-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/IFileResource.java b/...e-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/IFileResource.java
@@ -35,6 +35,8 @@ public interface IFileResource {
 
     RemoteClient getRemoteClient(BaseReq req);
 
+    ServiceResponse<Boolean> isFileAvailable(BaseReq req);
+
     ServiceResponse<FileNodesDTO> listFileNode(ListFileNodeReq req);
 
     ServiceResponse<Boolean> executeAction(ExecuteActionReq req);

diff --git a/...e-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceProxy.java b/...e-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceProxy.java
@@ -69,6 +69,11 @@ public RemoteClient getRemoteClient(BaseReq req) {
         return chooseFileResource(req).getRemoteClient(req);
     }
 
+    @Override
+    public ServiceResponse<Boolean> isFileAvailable(BaseReq req) {
+        return chooseFileResource(req).isFileAvailable(req);
+    }
+
     @Override
     public ServiceResponse<FileNodesDTO> listFileNode(ListFileNodeReq req) {
         return chooseFileResource(req).listFileNode(req);

diff --git a/...ain/java/com/tencent/bk/job/file/worker/artifactory/impl/ArtifactoryFileResourceImpl.java b/...ain/java/com/tencent/bk/job/file/worker/artifactory/impl/ArtifactoryFileResourceImpl.java
@@ -75,6 +75,12 @@ public RemoteClient getRemoteClient(BaseReq req) {
         return baseService.getArtifactoryClientFromBaseReq(req);
     }
 
+    @Override
+    public ServiceResponse<Boolean> isFileAvailable(BaseReq req) {
+        ArtifactoryRemoteClient client = baseService.getArtifactoryClientFromBaseReq(req);
+        return ServiceResponse.buildSuccessResp(client.isAvailable());
+    }
+
     private String parseParentNodeTypeByPath(String path) {
         if (StringUtils.isBlank(path)) return ArtifactoryNodeTypeEnum.FILE_SOURCE.name();
         path = StringUtil.removePrefix(path, "/");