diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java index 1e14fcb705..95af6ba1ca 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java @@ -64,7 +64,12 @@ public EsbIamClient(String esbHostUrl, String appCode, String appSecret, String @Override public String getApplyUrl(List actionList) { - GetApplyUrlRequest getApplyUrlRequest = makeBaseReqByWeb(GetApplyUrlRequest.class, null, "admin", "superadmin"); + GetApplyUrlRequest getApplyUrlRequest = makeBaseReqByWeb( + GetApplyUrlRequest.class, + null, + "admin", + "superadmin" + ); getApplyUrlRequest.setSystem(SystemId.JOB); getApplyUrlRequest.setAction(actionList); String respStr = null; @@ -136,7 +141,11 @@ public boolean registerResource(String id, String name, String type, String crea } @Override - public EsbIamAuthedPolicy authByPath(EsbIamAction esbIamAction, EsbIamSubject esbIamSubject, List esbIamResources) { + public EsbIamAuthedPolicy authByPath( + EsbIamAction esbIamAction, + EsbIamSubject esbIamSubject, + List esbIamResources + ) { AuthByPathReq authByPathReq = makeBaseReqByWeb(AuthByPathReq.class, null, "admin", "superadmin"); authByPathReq.setAction(esbIamAction); @@ -150,15 +159,23 @@ public EsbIamAuthedPolicy authByPath(EsbIamAction esbIamAction, EsbIamSubject es } @Override - public List batchAuthByPath(List esbIamActions, EsbIamSubject esbIamSubject, List esbIamBatchPathResources, Long expiredAt) { + public List batchAuthByPath( + List esbIamActions, + EsbIamSubject esbIamSubject, + List esbIamBatchPathResources, + Long expiredAt + ) { BatchAuthByPathReq batchAuthByPathReq = makeBaseReqByWeb(BatchAuthByPathReq.class, null, "admin", "superadmin"); batchAuthByPathReq.setActions(esbIamActions); batchAuthByPathReq.setSubject(esbIamSubject); batchAuthByPathReq.setResources(esbIamBatchPathResources); batchAuthByPathReq.setExpiredAt(expiredAt); - EsbResp> esbResp = getEsbRespByReq(HttpPost.METHOD_NAME, - API_BATCH_AUTH_BY_PATH_URL, batchAuthByPathReq, new TypeReference>>() { + EsbResp> esbResp = getEsbRespByReq( + HttpPost.METHOD_NAME, + API_BATCH_AUTH_BY_PATH_URL, + batchAuthByPathReq, + new TypeReference>>() { }); return esbResp.getData(); } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ReflectUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ReflectUtil.java index 277b91c535..8d8a538418 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ReflectUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ReflectUtil.java @@ -38,7 +38,7 @@ public static Object getFieldValue(Object obj, String propertyName) { field.setAccessible(true); return field.get(obj); } catch (NoSuchFieldException e) { - log.info("Fail to get field {} of {}", propertyName, obj); + log.info("no field {} of {}", propertyName, obj); } catch (IllegalAccessException e) { log.info("Cannot access field {} of {}", propertyName, obj); } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/StringUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/StringUtil.java index 550f79d45e..fbaff9c83f 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/StringUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/StringUtil.java @@ -30,7 +30,13 @@ import org.apache.commons.lang3.tuple.Pair; import java.net.URLEncoder; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -44,6 +50,7 @@ public class StringUtil { public static String replacePathVariables(String rawStr, Object obj) { + if (obj == null) return rawStr; List placeholderList = findOneRegexPatterns(rawStr, "(\\{.*?\\})"); for (String placeholder : placeholderList) { String fieldName = placeholder.substring(1, placeholder.length() - 1); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java index 619901e508..7afbffbda3 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.ServiceResponse; +import com.tencent.bk.job.manage.model.inner.ServiceAppBaseInfoDTO; import com.tencent.bk.job.manage.model.inner.ServiceApplicationDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostStatusDTO; import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByDynamicGroupReq; @@ -49,6 +50,14 @@ @RestController @InternalAPI public interface ServiceApplicationResource { + /** + * 查询CMDB中的常规业务列表 + * + * @return + */ + @RequestMapping("/list/normal") + ServiceResponse> listNormalApps(); + /** * 根据业务id查询业务 * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/CompositeExpression.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAppBaseInfoDTO.java similarity index 80% rename from src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/CompositeExpression.java rename to src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAppBaseInfoDTO.java index 37e9955d19..5a8db05f8b 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/CompositeExpression.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAppBaseInfoDTO.java @@ -22,18 +22,25 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.upgrader.model; +package com.tencent.bk.job.manage.model.inner; -import lombok.AllArgsConstructor; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; import lombok.Data; -import lombok.NoArgsConstructor; -import java.util.List; - -@NoArgsConstructor -@AllArgsConstructor +/** + * 业务 + */ @Data -public class CompositeExpression { - private String op; - private List content; +@ApiModel("业务") +public class ServiceAppBaseInfoDTO { + + @ApiModelProperty("业务ID") + private Long id; + + /** + * 业务名称 + */ + @ApiModelProperty("业务名称") + private String name; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java index 6546df7dca..277fb52c4b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java @@ -31,6 +31,7 @@ import com.tencent.bk.job.common.model.vo.HostInfoVO; import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; import com.tencent.bk.job.manage.dao.ApplicationInfoDAO; +import com.tencent.bk.job.manage.model.inner.ServiceAppBaseInfoDTO; import com.tencent.bk.job.manage.model.inner.ServiceApplicationDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostStatusDTO; import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByDynamicGroupReq; @@ -60,6 +61,14 @@ public ServiceApplicationResourceImpl(ApplicationService applicationService, this.applicationInfoDAO = applicationInfoDAO; } + @Override + public ServiceResponse> listNormalApps() { + List applicationInfoDTOList = applicationInfoDAO.listAppInfoByType(AppTypeEnum.NORMAL); + List resultList = + applicationInfoDTOList.parallelStream().map(this::convertToServiceAppBaseInfo).collect(Collectors.toList()); + return ServiceResponse.buildSuccessResp(resultList); + } + @Override public ServiceApplicationDTO queryAppById(Long appId) { ApplicationInfoDTO appInfo = applicationService.getAppInfoById(appId); @@ -83,6 +92,12 @@ private ServiceApplicationDTO convertToServiceApp(ApplicationInfoDTO appInfo) { return app; } + private ServiceAppBaseInfoDTO convertToServiceAppBaseInfo(ApplicationInfoDTO appInfo) { + ServiceAppBaseInfoDTO appBaseInfoDTO = new ServiceAppBaseInfoDTO(); + appBaseInfoDTO.setId(appInfo.getId()); + appBaseInfoDTO.setName(appInfo.getName()); + return appBaseInfoDTO; + } @Override public ServiceResponse checkAppPermission(Long appId, String username) { diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/Upgrader.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/Upgrader.java index ee8e4ff2b5..d352fb259f 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/Upgrader.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/Upgrader.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.upgrader; import com.tencent.bk.job.common.util.CompareUtil; +import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; import com.tencent.bk.job.upgrader.anotation.RequireTaskParam; import com.tencent.bk.job.upgrader.anotation.UpgradeTask; @@ -40,9 +41,11 @@ import org.reflections.scanners.TypeAnnotationsScanner; import java.io.BufferedReader; -import java.io.FileReader; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.lang.reflect.Constructor; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -68,8 +71,8 @@ public static void usage() { "then enter the specific parameters related to the " + "specific upgrade tasks according to the command line prompt"); log.info("/path/to/log/dir is the path to log dir, usually set to ${BK_HOME}/logs/job"); - log.info("/path/to/config/file is the path to config file, which is generated by upgrade script" + - "路径为${BK_HOME}/etc/job/upgrader/upgrader.properties"); + log.info("/path/to/config/file is the path to config file, which is generated by upgrade script, " + + "usually can be ${BK_HOME}/etc/job/upgrader/upgrader.properties"); log.info("fromVersion is the current version of Job, example:3.2.7.3"); log.info("toVersion is the target version of Job to upgrade, example:3.3.4.0"); log.info("executeTime is the time point to execute upgrade tasks, " + @@ -112,43 +115,37 @@ public static boolean isTaskNeedToExecute( && CompareUtil.compareVersion(taskTargetVersion, toVersion) <= 0; } - public static void main(String[] args) { - log.info("Upgrader begin to run"); - if (args.length < 3) { - usage(); - return; - } - String fromVersion = args[0]; - String toVersion = args[1]; - String executeTime = args[2]; - log.info("fromVersion={}", fromVersion); - log.info("toVersion={}", toVersion); - log.info("executeTime={}", executeTime); - - Properties properties = new Properties(); - String configFilePath = System.getProperty("config.file"); - if (StringUtils.isNotBlank(configFilePath)) { - try { - properties.load(new BufferedReader(new FileReader(configFilePath))); - } catch (IOException e) { - log.warn("Cannot read configFile from path:{}, exit", configFilePath, e); - return; - } - } else { - log.warn("Config file is empty"); - return; - } - log.info("Upgrader begin to run"); + private static List, String, Integer>> findAndFilterUpgradeTasks( + String fromVersion, + String toVersion, + String executeTime + ) { + List, String, Integer>> upgradeTaskList = new ArrayList<>(); // 找出所有UpgradeTask Reflections reflections = new Reflections( "com.tencent.bk.job.upgrader.task", new SubTypesScanner(false), new TypeAnnotationsScanner() ); - List, String, Integer>> upgradeTaskList = new ArrayList<>(); Set> upgradeTaskSet = reflections.getTypesAnnotatedWith(UpgradeTask.class); + + // 明确指定运行的任务 + List targetTaskList = null; + String targetTasksStr = System.getProperty("target.tasks"); + if (StringUtils.isNotBlank(targetTasksStr)) { + targetTasksStr = targetTasksStr.trim(); + targetTaskList = StringUtil.strToList(targetTasksStr, String.class, ","); + log.info("targetTaskList={}", targetTaskList); + } + // 筛选 - upgradeTaskSet.forEach(clazz -> { + for (Class clazz : upgradeTaskSet) { + if (targetTaskList != null && !targetTaskList.isEmpty()) { + // 只运行指定的任务 + if (!targetTaskList.contains(clazz.getSimpleName())) { + continue; + } + } UpgradeTask anotation = clazz.getAnnotation(UpgradeTask.class); String dataStartVersion = anotation.dataStartVersion(); String targetVersion = anotation.targetVersion(); @@ -157,7 +154,8 @@ public static void main(String[] args) { log.info("Found upgradeTask:[{}] for version {}, dataStartVersion={}, priority={}", clazz.getName(), targetVersion, dataStartVersion, priority); if (targetExecuteTime.name().equalsIgnoreCase(executeTime) - && isTaskNeedToExecute(fromVersion, toVersion, dataStartVersion, targetVersion)) { + && isTaskNeedToExecute(fromVersion, toVersion, dataStartVersion, targetVersion) + ) { log.info("{}-->{},{},Add task {}({},{})", fromVersion, toVersion, executeTime, clazz.getSimpleName(), targetVersion, targetExecuteTime); upgradeTaskList.add(Triple.of(clazz, targetVersion, priority)); @@ -165,18 +163,14 @@ && isTaskNeedToExecute(fromVersion, toVersion, dataStartVersion, targetVersion)) log.info("{}-->{},{},Ignore task {}({},{})", fromVersion, toVersion, executeTime, clazz.getSimpleName(), targetVersion, targetExecuteTime); } - }); - // 排序 - upgradeTaskList.sort((o1, o2) -> { - int result = CompareUtil.compareVersion(o1.getMiddle(), o2.getMiddle()); - if (result != 0) return result; - return o1.getRight().compareTo(o2.getRight()); - }); - log.info("upgradeTaskList after sort:"); - upgradeTaskList.forEach(entry -> { - log.info("[{}] for version {}, priority={}", entry.getLeft(), entry.getMiddle(), entry.getRight()); - }); - // 参数输入 + } + return upgradeTaskList; + } + + private static void checkAndInputTaskParams( + List, String, Integer>> upgradeTaskList, + Properties properties + ) { Map paramMap = new HashMap<>(); Scanner scanner = new Scanner(System.in); for (Triple, String, Integer> entry : upgradeTaskList) { @@ -194,8 +188,8 @@ && isTaskNeedToExecute(fromVersion, toVersion, dataStartVersion, targetVersion)) AbstractTaskParam paramInstance = (AbstractTaskParam) paramClass.newInstance(); log.info("Param {}", i + 1); log.info("Name: " + paramInstance.getKey()); - log.info("描述：" + paramInstance.getDescription()); log.info("Description: " + paramInstance.getDescriptionEn()); + log.info("描述：" + paramInstance.getDescription()); // 从JVM参数中取值 String valueBySystemProperty = System.getProperty(paramInstance.getKey()); if (!StringUtils.isBlank(valueBySystemProperty)) { @@ -215,6 +209,7 @@ && isTaskNeedToExecute(fromVersion, toVersion, dataStartVersion, targetVersion)) } // 都没有再要求输入 log.info("Please input param value and click enter to continue:"); + log.info("请输入上述提示所描述的参数，按回车键确认输入："); AbstractTaskParam.ParamCheckResult paramCheckResult; String paramValue; do { @@ -235,6 +230,62 @@ && isTaskNeedToExecute(fromVersion, toVersion, dataStartVersion, targetVersion)) } } paramMap.forEach(properties::setProperty); + } + + public static void main(String[] args) { + log.info("Upgrader begin to run"); + if (args.length < 3) { + usage(); + return; + } + String fromVersion = args[0]; + String toVersion = args[1]; + String executeTime = args[2]; + log.info("fromVersion={}", fromVersion); + log.info("toVersion={}", toVersion); + log.info("executeTime={}", executeTime); + + Properties properties = new Properties(); + String configFilePath = System.getProperty("config.file"); + if (StringUtils.isNotBlank(configFilePath)) { + BufferedReader br = null; + try { + br = new BufferedReader( + new InputStreamReader(new FileInputStream(configFilePath), StandardCharsets.UTF_8) + ); + properties.load(br); + } catch (IOException e) { + log.warn("Cannot read configFile from path:{}, exit", configFilePath, e); + return; + } finally { + if (br != null) { + try { + br.close(); + } catch (IOException e) { + log.warn("Fail to close br", e); + } + } + } + } else { + log.warn("Config file is empty"); + return; + } + + List, String, Integer>> upgradeTaskList = + findAndFilterUpgradeTasks(fromVersion, toVersion, executeTime); + + // 排序 + upgradeTaskList.sort((o1, o2) -> { + int result = CompareUtil.compareVersion(o1.getMiddle(), o2.getMiddle()); + if (result != 0) return result; + return o1.getRight().compareTo(o2.getRight()); + }); + log.info("upgradeTaskList after sort:"); + upgradeTaskList.forEach(entry -> { + log.info("[{}] for version {}, priority={}", entry.getLeft(), entry.getMiddle(), entry.getRight()); + }); + // 参数输入 + checkAndInputTaskParams(upgradeTaskList, properties); // 运行 runTasks(upgradeTaskList, args, properties); } @@ -252,6 +303,7 @@ private static int runTasks( currentTaskNum.incrementAndGet(); Class clazz = entry.getLeft(); IUpgradeTask upgradeTask = getUpgradeTaskByClass(clazz, properties); + upgradeTask.init(); AtomicBoolean taskSuccess = new AtomicBoolean(false); Thread taskThread = new Thread() { @Override diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractHttpClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractHttpClient.java new file mode 100644 index 0000000000..bf17e1f71c --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractHttpClient.java @@ -0,0 +1,126 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.upgrader.client; + +import com.tencent.bk.job.common.util.http.AbstractHttpHelper; +import com.tencent.bk.job.common.util.http.BasicHttpReq; +import com.tencent.bk.job.common.util.http.DefaultHttpHelper; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.upgrader.model.IamReq; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; + +import java.util.List; + +import static com.tencent.bk.job.common.constant.HttpHeader.HDR_CONTENT_TYPE; + +@Slf4j +public abstract class AbstractHttpClient { + private String hostUrl; + private AbstractHttpHelper defaultHttpHelper = new DefaultHttpHelper(); + + public AbstractHttpClient(String hostUrl) { + this.hostUrl = hostUrl; + } + + public String doHttpGet(String uri, IamReq params) throws Exception { + return doHttpGet(uri, params, defaultHttpHelper); + } + + abstract List
getBasicHeaders(); + + public String doHttpGet(String uri, BasicHttpReq params, AbstractHttpHelper httpHelper) throws Exception { + if (params == null) { + params = new BasicHttpReq(); + } + if (httpHelper == null) { + httpHelper = defaultHttpHelper; + } + boolean error = false; + long start = System.currentTimeMillis(); + String responseBody = null; + String url = hostUrl; + try { + if (!hostUrl.endsWith("/") && !uri.startsWith("/")) { + url = hostUrl + "/" + uri + params.toUrlParams(); + } else { + url = hostUrl + uri + params.toUrlParams(); + } + responseBody = httpHelper.get(url, getBasicHeaders()); + return responseBody; + } catch (Exception e) { + log.warn("Get url {}| params={}| exception={}", hostUrl + uri, + JsonUtils.toJsonWithoutSkippedFields(params), + e.getMessage()); + error = true; + throw e; + } finally { + log.info("Get url {}| error={}| params={}| time={}| resp={}", hostUrl + uri, error, + JsonUtils.toJsonWithoutSkippedFields(params), (System.currentTimeMillis() - start), responseBody); + } + } + + protected String doHttpPost(String uri, T params) throws Exception { + return doHttpPost(uri, params, defaultHttpHelper); + } + + protected String doHttpPost( + String uri, T params, + AbstractHttpHelper httpHelper + ) throws Exception { + if (httpHelper == null) { + httpHelper = defaultHttpHelper; + } + boolean error = false; + long start = System.currentTimeMillis(); + String responseBody = null; + try { + String url; + if (!hostUrl.endsWith("/") && !uri.startsWith("/")) { + url = hostUrl + "/" + uri; + } else { + url = hostUrl + uri; + } + List
headerList = getBasicHeaders(); + headerList.add(new BasicHeader(HDR_CONTENT_TYPE, "application/json")); + responseBody = httpHelper.post(url, "UTF-8", buildPostBody(params), headerList); + return responseBody; + } catch (Exception e) { + log.warn("Post url {}| params={}| exception={}", uri, JsonUtils.toJsonWithoutSkippedFields(params), + e.getMessage()); + error = true; + throw e; + } finally { + log.info("Post url {}| error={}| params={}| time={}| resp={}", uri, error, + JsonUtils.toJsonWithoutSkippedFields(params), (System.currentTimeMillis() - start), responseBody); + } + } + + protected String buildPostBody(T params) { + return JsonUtils.toJson(params); + } + +} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractIamClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractIamClient.java index f2d150f505..68d9abdd2a 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractIamClient.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractIamClient.java @@ -24,10 +24,6 @@ package com.tencent.bk.job.upgrader.client; -import com.tencent.bk.job.common.util.http.AbstractHttpHelper; -import com.tencent.bk.job.common.util.http.DefaultHttpHelper; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.upgrader.model.IamReq; import lombok.extern.slf4j.Slf4j; import org.apache.http.Header; import org.apache.http.message.BasicHeader; @@ -35,26 +31,19 @@ import java.util.ArrayList; import java.util.List; -import static com.tencent.bk.job.common.constant.HttpHeader.HDR_CONTENT_TYPE; - @Slf4j -public abstract class AbstractIamClient { - private String iamHostUrl; +public abstract class AbstractIamClient extends AbstractHttpClient { private String appSecret; private String appCode; - private AbstractHttpHelper defaultHttpHelper = new DefaultHttpHelper(); public AbstractIamClient(String iamHostUrl, String appCode, String appSecret) { - this.iamHostUrl = iamHostUrl; + super(iamHostUrl); this.appCode = appCode; this.appSecret = appSecret; } - public String doHttpGet(String uri, IamReq params) throws Exception { - return doHttpGet(uri, params, defaultHttpHelper); - } - - private List
getBasicHeaders() { + @Override + protected List
getBasicHeaders() { List
headerList = new ArrayList<>(); headerList.add(new BasicHeader("X-Bk-App-Code", appCode)); headerList.add(new BasicHeader("X-Bk-App-Secret", appSecret)); @@ -62,72 +51,4 @@ private List
getBasicHeaders() { return headerList; } - public String doHttpGet(String uri, IamReq params, AbstractHttpHelper httpHelper) throws Exception { - if (httpHelper == null) { - httpHelper = defaultHttpHelper; - } - boolean error = false; - long start = System.currentTimeMillis(); - String responseBody = null; - String url = iamHostUrl; - try { - if (!iamHostUrl.endsWith("/") && !uri.startsWith("/")) { - url = iamHostUrl + "/" + uri + params.toUrlParams(); - } else { - url = iamHostUrl + uri + params.toUrlParams(); - } - responseBody = httpHelper.get(url, getBasicHeaders()); - return responseBody; - } catch (Exception e) { - log.warn("Get url {}| params={}| exception={}", iamHostUrl + uri, - JsonUtils.toJsonWithoutSkippedFields(params), - e.getMessage()); - error = true; - throw e; - } finally { - log.info("Get url {}| error={}| params={}| time={}| resp={}", iamHostUrl + uri, error, - JsonUtils.toJsonWithoutSkippedFields(params), (System.currentTimeMillis() - start), responseBody); - } - } - - protected String doHttpPost(String uri, T params) throws Exception { - return doHttpPost(uri, params, defaultHttpHelper); - } - - protected String doHttpPost( - String uri, T params, - AbstractHttpHelper httpHelper - ) throws Exception { - if (httpHelper == null) { - httpHelper = defaultHttpHelper; - } - boolean error = false; - long start = System.currentTimeMillis(); - String responseBody = null; - try { - String url; - if (!iamHostUrl.endsWith("/") && !uri.startsWith("/")) { - url = iamHostUrl + "/" + uri; - } else { - url = iamHostUrl + uri; - } - List
headerList = getBasicHeaders(); - headerList.add(new BasicHeader(HDR_CONTENT_TYPE, "application/json")); - responseBody = httpHelper.post(url, "UTF-8", buildPostBody(params), headerList); - return responseBody; - } catch (Exception e) { - log.warn("Post url {}| params={}| exception={}", uri, JsonUtils.toJsonWithoutSkippedFields(params), - e.getMessage()); - error = true; - throw e; - } finally { - log.info("Post url {}| error={}| params={}| time={}| resp={}", uri, error, - JsonUtils.toJsonWithoutSkippedFields(params), (System.currentTimeMillis() - start), responseBody); - } - } - - protected String buildPostBody(T params) { - return JsonUtils.toJson(params); - } - } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractJobClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractJobClient.java new file mode 100644 index 0000000000..b7ea9a001b --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractJobClient.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.upgrader.client; + +import lombok.extern.slf4j.Slf4j; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public abstract class AbstractJobClient extends AbstractHttpClient { + private String jobAuthToken; + + public AbstractJobClient(String jobHostUrl, String jobAuthToken) { + super(jobHostUrl); + this.jobAuthToken = jobAuthToken; + } + + @Override + protected List
getBasicHeaders() { + List
headerList = new ArrayList<>(); + headerList.add(new BasicHeader("x-job-auth-token", jobAuthToken)); + headerList.add(new BasicHeader("Content-Type", "application/json")); + return headerList; + } + +} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/JobClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/JobClient.java new file mode 100644 index 0000000000..ef20fdc4fc --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/JobClient.java @@ -0,0 +1,141 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.upgrader.client; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.model.ServiceResponse; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.http.AbstractHttpHelper; +import com.tencent.bk.job.common.util.http.BasicHttpReq; +import com.tencent.bk.job.common.util.json.JsonMapper; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.upgrader.model.AppInfo; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; + +import java.util.List; + +/** + * Job接口调用客户端 + */ +@Slf4j +public class JobClient extends AbstractJobClient { + + /** + * Job API 处理请求成功 + */ + private static final int RESULT_OK = 0; + + private static final String URL_LIST_NORMAL_APPS = "/service/app/list/normal"; + + private static final JsonMapper JSON_MAPPER = JsonMapper.nonDefaultMapper(); + + public JobClient(String jobHostUrl, String jobAuthToken) { + super(jobHostUrl, jobAuthToken); + } + + private R getJobRespByReq(String method, String uri, BasicHttpReq reqBody, + TypeReference typeReference) { + return getJobRespByReq(method, uri, reqBody, typeReference, null); + } + + @SuppressWarnings("all") + private R getJobRespByReq( + String method, + String uri, + BasicHttpReq reqBody, + TypeReference typeReference, + AbstractHttpHelper httpHelper + ) { + // URL模板变量替换 + uri = StringUtil.replacePathVariables(uri, reqBody); + String reqStr = JsonUtils.toJsonWithoutSkippedFields(reqBody); + String respStr = null; + try { + if (method.equals(HttpGet.METHOD_NAME)) { + respStr = doHttpGet(uri, reqBody, httpHelper); + } else if (method.equals(HttpPost.METHOD_NAME)) { + respStr = doHttpPost(uri, reqBody, httpHelper); + } + if (StringUtils.isBlank(respStr)) { + log.error("fail:response is blank|method={}|uri={}|reqStr={}", method, uri, reqStr); + throw new ServiceException(ErrorCode.SERVICE_INTERNAL_ERROR, "response is blank"); + } else { + log.debug("success|method={}|uri={}|reqStr={}|respStr={}", method, uri, reqStr, respStr); + } + R result = + JSON_MAPPER.fromJson(respStr, typeReference); + ServiceResponse jobResp = (ServiceResponse) result; + if (jobResp == null) { + log.error("fail:jobResp is null after parse|method={}|uri={}|reqStr={}|respStr={}", method, uri, + reqStr, respStr); + throw new ServiceException(ErrorCode.SERVICE_INTERNAL_ERROR, "jobResp is null after parse"); + } else if (jobResp.getCode() != RESULT_OK) { + log.error( + "fail:jobResp code!=0|jobResp.code={}|jobResp" + + ".errorMessage={}|method={}|uri={}|reqStr={}|respStr={}", + jobResp.getCode(), + jobResp.getErrorMsg(), + method, + uri, + reqStr, + respStr + ); + throw new ServiceException(ErrorCode.SERVICE_INTERNAL_ERROR, "jobResp code!=0"); + } + if (jobResp.getData() == null) { + log.warn( + "warn:jobResp.getData() == null|jobResp.code={}|jobResp" + + ".errorMessage={}|method={}|uri={}|reqStr={}|respStr={}", + jobResp.getCode(), + jobResp.getErrorMsg(), + method, + uri, + reqStr, + respStr + ); + } + return result; + } catch (Exception e) { + String errorMsg = "Fail to request JOB data|method=" + method + "|uri=" + uri + "|reqStr=" + reqStr; + log.error(errorMsg, e); + throw new ServiceException(ErrorCode.SERVICE_INTERNAL_ERROR, "Fail to request JOB data"); + } + } + + public List listNormalApps() { + ServiceResponse> resp = getJobRespByReq( + HttpGet.METHOD_NAME, + URL_LIST_NORMAL_APPS, + new BasicHttpReq(), + new TypeReference>>() { + }); + return resp.getData(); + } +} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/iam/JobIamHelper.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/iam/JobIamHelper.java new file mode 100644 index 0000000000..220810ec82 --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/iam/JobIamHelper.java @@ -0,0 +1,95 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.upgrader.iam; + +import com.tencent.bk.job.common.iam.config.EsbConfiguration; +import com.tencent.bk.job.common.iam.util.BusinessAuthHelper; +import com.tencent.bk.sdk.iam.config.IamConfiguration; +import com.tencent.bk.sdk.iam.constants.SystemId; +import com.tencent.bk.sdk.iam.helper.AuthHelper; +import com.tencent.bk.sdk.iam.service.HttpClientService; +import com.tencent.bk.sdk.iam.service.PolicyService; +import com.tencent.bk.sdk.iam.service.TokenService; +import com.tencent.bk.sdk.iam.service.impl.DefaultHttpClientServiceImpl; +import com.tencent.bk.sdk.iam.service.impl.PolicyServiceImpl; +import com.tencent.bk.sdk.iam.service.impl.TokenServiceImpl; + +public class JobIamHelper { + + /** + * ESB 分配的系统 App Code + */ + private String appCode; + + /** + * ESB 分配的系统 App Secret + */ + private String appSecret; + + /** + * 权限中心的访问地址 + */ + private String iamBaseUrl; + + /** + * ESB API 的访问地址 + */ + private String esbUrl; + + public JobIamHelper(String appCode, String appSecret, String iamBaseUrl, String esbUrl) { + this.appCode = appCode; + this.appSecret = appSecret; + this.iamBaseUrl = iamBaseUrl; + this.esbUrl = esbUrl; + } + + public IamConfiguration iamConfiguration() { + return new IamConfiguration(SystemId.JOB, appCode, appSecret, iamBaseUrl); + } + + public EsbConfiguration esbConfiguration() { + return new EsbConfiguration(esbUrl, false); + } + + public HttpClientService httpClientService() { + return new DefaultHttpClientServiceImpl(iamConfiguration()); + } + + public PolicyService policyService() { + return new PolicyServiceImpl(iamConfiguration(), httpClientService()); + } + + public TokenService tokenService() { + return new TokenServiceImpl(iamConfiguration(), httpClientService()); + } + + public AuthHelper authHelper() { + return new AuthHelper(tokenService(), policyService(), iamConfiguration()); + } + + public BusinessAuthHelper businessAuthHelper() { + return new BusinessAuthHelper(tokenService(), policyService(), iamConfiguration()); + } +} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Expression.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/AppInfo.java similarity index 94% rename from src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Expression.java rename to src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/AppInfo.java index bbdb59e72a..389c440ddb 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Expression.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/AppInfo.java @@ -28,11 +28,11 @@ import lombok.Data; import lombok.NoArgsConstructor; +@Data @NoArgsConstructor @AllArgsConstructor -@Data -public class Expression { - private String field; - private String op; - private String value; +public class AppInfo { + private Long id; + + private String name; } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Policy.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Policy.java index 33225ca323..8a14d3ea14 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Policy.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Policy.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.upgrader.model; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.sdk.iam.dto.expression.ExpressionDTO; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -36,7 +37,7 @@ public class Policy { private String version; private Long id; private Subject subject; - private CompositeExpression expression; + private ExpressionDTO expression; @JsonProperty("expired_at") private Long expiredAt; } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java index b9a31a29e9..bc6203e605 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java @@ -39,10 +39,14 @@ public abstract class BaseUpgradeTask implements IUpgradeTask { this.properties = properties; } - public Properties getProperties(){ + public Properties getProperties() { return properties; } + @Override + public void init() { + } + @Override public String getName() { return this.getClass().getSimpleName(); diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/IUpgradeTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/IUpgradeTask.java index 23cd3dab12..fef6fd92a8 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/IUpgradeTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/IUpgradeTask.java @@ -32,6 +32,8 @@ public interface IUpgradeTask { String getTargetVersion(); + void init(); + int getPriority(); int execute(String[] args); diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/UseAccountPermissionMigrationTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/UseAccountPermissionMigrationTask.java new file mode 100644 index 0000000000..e767c78b5b --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/UseAccountPermissionMigrationTask.java @@ -0,0 +1,310 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.upgrader.task; + +import com.tencent.bk.job.common.iam.client.EsbIamClient; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeEnum; +import com.tencent.bk.job.common.iam.dto.EsbIamAction; +import com.tencent.bk.job.common.iam.dto.EsbIamBatchAuthedPolicy; +import com.tencent.bk.job.common.iam.dto.EsbIamBatchPathResource; +import com.tencent.bk.job.common.iam.dto.EsbIamPathItem; +import com.tencent.bk.job.common.iam.dto.EsbIamSubject; +import com.tencent.bk.job.common.iam.util.BusinessAuthHelper; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; +import com.tencent.bk.job.upgrader.anotation.RequireTaskParam; +import com.tencent.bk.job.upgrader.anotation.UpgradeTask; +import com.tencent.bk.job.upgrader.anotation.UpgradeTaskInputParam; +import com.tencent.bk.job.upgrader.client.IamClient; +import com.tencent.bk.job.upgrader.client.JobClient; +import com.tencent.bk.job.upgrader.iam.JobIamHelper; +import com.tencent.bk.job.upgrader.model.ActionPolicies; +import com.tencent.bk.job.upgrader.model.AppInfo; +import com.tencent.bk.job.upgrader.model.Policy; +import com.tencent.bk.job.upgrader.task.param.JobManageServerAddress; +import com.tencent.bk.job.upgrader.task.param.ParamNameConsts; +import com.tencent.bk.sdk.iam.constants.SystemId; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +/** + * 账号使用权限迁移任务 + */ +@Slf4j +@RequireTaskParam(value = { + @UpgradeTaskInputParam(value = JobManageServerAddress.class) +}) +@UpgradeTask( + dataStartVersion = "3.0.0.0", + targetVersion = "3.3.5.0", + targetExecuteTime = ExecuteTimeEnum.AFTER_UPDATE_JOB) +public class UseAccountPermissionMigrationTask extends BaseUpgradeTask { + + private JobIamHelper jobIamHelper; + private JobClient jobManageClient; + + private IamClient iamClient; + private EsbIamClient esbIamClient; + private List appInfoList; + private Map appInfoMap; + + private String getJobHostUrlByAddress(String address) { + if (!address.startsWith("http://") && !address.startsWith("https://")) { + address = "http://" + address; + } + return address; + } + + public UseAccountPermissionMigrationTask(Properties properties) { + super(properties); + } + + @Override + public void init() { + jobIamHelper = new JobIamHelper( + (String) getProperties().get(ParamNameConsts.CONFIG_PROPERTY_APP_CODE), + (String) getProperties().get(ParamNameConsts.CONFIG_PROPERTY_APP_SECRET), + (String) getProperties().get(ParamNameConsts.CONFIG_PROPERTY_IAM_BASE_URL), + (String) getProperties().get(ParamNameConsts.CONFIG_PROPERTY_ESB_SERVICE_URL) + ); + jobManageClient = new JobClient( + getJobHostUrlByAddress((String) getProperties().get(ParamNameConsts.INPUT_PARAM_JOB_MANAGE_SERVER_ADDRESS)), + (String) getProperties().get(ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PUBLIC_KEY_BASE64) + ); + this.appInfoList = getAllNormalAppInfoFromManage(); + appInfoMap = new HashMap<>(); + appInfoList.forEach(appInfo -> { + appInfoMap.put(appInfo.getId(), appInfo.getName()); + }); + } + + private IamClient getIamClient() { + Properties properties = getProperties(); + if (iamClient == null) { + iamClient = new IamClient( + (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_IAM_BASE_URL), + (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_CODE), + (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_SECRET) + ); + } + return iamClient; + } + + private EsbIamClient getEsbIamClient() { + Properties properties = getProperties(); + if (esbIamClient == null) { + esbIamClient = new EsbIamClient( + (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_ESB_SERVICE_URL), + (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_CODE), + (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_SECRET), + false + ); + } + return esbIamClient; + } + + private List queryAuthorizedPolicies(String actionId) { + ActionPolicies actionPolicies = getIamClient().getActionPolicies(actionId); + return actionPolicies.getResults(); + } + + private List getAllNormalAppInfoFromManage() { + try { + return jobManageClient.listNormalApps(); + } catch (Exception e) { + log.error("Fail to get normal apps from job-manage, please confirm job-manage version>=3.3.5.0"); + throw e; + } + } + + /** + * 根据策略计算出有权限的业务Id列表 + * + * @param policy + * @return + */ + private List getAuthorizedAppIdList(Policy policy) { + BusinessAuthHelper businessAuthHelper = jobIamHelper.businessAuthHelper(); + return businessAuthHelper.getAuthedAppIdList( + null, + policy.getExpression(), + appInfoList.parallelStream().map(AppInfo::getId).collect(Collectors.toList()) + ); + } + + /** + * 根据业务ID获取业务名称 + * + * @param appId + * @return + */ + private String getAppNameById(Long appId) { + if (appInfoMap.containsKey(appId)) return appInfoMap.get(appId); + return null; + } + + private boolean authByPolicy(Policy policy) { + if ("any".equals(policy.getExpression().getOperator().getOperator())) { + // 授予任意业务任意账号权限 + log.info("auth any biz permission to {}", policy.getSubject()); + return authAnyBizUseAccountByPolicy(policy); + } else { + List appIdList = getAuthorizedAppIdList(policy); + log.info("auth {} permission to {}", appIdList, policy.getSubject()); + // 授予业务下任意账号权限 + return authBizUseAccountByPolicy(policy, appIdList); + } + } + + private List getUseAccountActions() { + EsbIamAction esbIamAction = new EsbIamAction(); + esbIamAction.setId(ActionId.USE_ACCOUNT); + List actions = new ArrayList<>(); + actions.add(esbIamAction); + return actions; + } + + private EsbIamSubject getSubjectByPolicy(Policy policy) { + EsbIamSubject esbIamSubject = new EsbIamSubject(); + esbIamSubject.setId(policy.getSubject().getId()); + esbIamSubject.setType(policy.getSubject().getType()); + return esbIamSubject; + } + + /** + * 授予任意业务下任意执行账号使用的权限 + * + * @param policy 权限策略 + * @return 是否授权成功 + */ + private boolean authAnyBizUseAccountByPolicy(Policy policy) { + EsbIamBatchPathResource esbIamBatchPathResource = new EsbIamBatchPathResource(); + esbIamBatchPathResource.setSystem(SystemId.JOB); + esbIamBatchPathResource.setType(ResourceTypeEnum.ACCOUNT.getId()); + esbIamBatchPathResource.setPaths(new ArrayList<>()); + return batchAuth(policy, getUseAccountActions(), getSubjectByPolicy(policy), esbIamBatchPathResource); + } + + /** + * 授予某些业务下任意执行账号使用的权限 + * + * @param policy 权限策略 + * @return 是否授权成功 + */ + private boolean authBizUseAccountByPolicy(Policy policy, List appIdList) { + EsbIamBatchPathResource esbIamBatchPathResource = new EsbIamBatchPathResource(); + esbIamBatchPathResource.setSystem(SystemId.JOB); + esbIamBatchPathResource.setType(ResourceTypeEnum.ACCOUNT.getId()); + List> pathList = new ArrayList<>(); + for (Long appId : appIdList) { + List esbIamPathItemList = new ArrayList<>(); + EsbIamPathItem pathItem = new EsbIamPathItem(); + pathItem.setType(ResourceTypeEnum.BUSINESS.getId()); + pathItem.setId(appId.toString()); + pathItem.setName(getAppNameById(appId)); + esbIamPathItemList.add(pathItem); + pathList.add(esbIamPathItemList); + } + esbIamBatchPathResource.setPaths(pathList); + return batchAuth(policy, getUseAccountActions(), getSubjectByPolicy(policy), esbIamBatchPathResource); + } + + /** + * 调用权限中心接口进行批量授权 + * + * @param policy 权限策略 + * @param actions 授权操作列表 + * @param esbIamSubject 授权对象 + * @param esbIamBatchPathResource 批量资源路径 + * @return 是否授权成功 + */ + private boolean batchAuth( + Policy policy, + List actions, + EsbIamSubject esbIamSubject, + EsbIamBatchPathResource esbIamBatchPathResource + ) { + List resourceList = new ArrayList<>(); + resourceList.add(esbIamBatchPathResource); + try { + List batchAuthedPolicy = getEsbIamClient().batchAuthByPath(actions, + esbIamSubject, resourceList, policy.getExpiredAt()); + log.info("batchAuthedPolicy={}", JsonUtils.toJson(batchAuthedPolicy)); + return true; + } catch (Exception e) { + log.error("Fail to auth subject {} to {}", JsonUtils.toJson(policy.getSubject()), policy.getExpiredAt(), e); + return false; + } + } + + public void printSeparateLine() { + log.info("=================================================="); + } + + public void showPolicies(List policies) { + policies.forEach(policy -> { + log.info("{}: {} expiredAt {}, expression:{}", policy.getId(), + policy.getSubject().getType() + ":" + policy.getSubject().getName(), + policy.getExpiredAt(), JsonUtils.toJson(policy.getExpression())); + }); + } + + @Override + public int execute(String[] args) { + log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); + String oldActionId = ActionId.LIST_BUSINESS; + // 1.旧权限数据读取 + List oldAuthorizedPolicies = queryAuthorizedPolicies(oldActionId); + printSeparateLine(); + log.info("oldAuthorizedPolicies of {}:", oldActionId); + showPolicies(oldAuthorizedPolicies); + printSeparateLine(); + log.info("Begin to auth according to oldPolicies:"); + // 2.新权限数据授权 + oldAuthorizedPolicies.forEach(policy -> { + log.info( + "auth {}:{}:{}:{}", + policy.getSubject().getType(), + policy.getSubject().getName(), + policy.getExpiredAt(), + authByPolicy(policy)); + }); + // 3.新权限策略查询 + String newActionId = ActionId.USE_ACCOUNT; + List newAuthorizedPolicies = queryAuthorizedPolicies(newActionId); + printSeparateLine(); + log.info("newAuthorizedPolicies:"); + showPolicies(newAuthorizedPolicies); + printSeparateLine(); + return 0; + } +}