feat: 执行引擎任务调度配额限制 TencentBlueKing#261

wangyu096 · Jul 2, 2024 · 124bf9b · 124bf9b
1 parent 92dba0f
commit 124bf9b
Show file tree

Hide file tree

Showing 19 changed files with 96 additions and 89 deletions.
diff --git a/...c/main/java/com/tencent/bk/job/common/service/feature/FeatureToggleAutoConfiguration.java b/...c/main/java/com/tencent/bk/job/common/service/feature/FeatureToggleAutoConfiguration.java
@@ -24,13 +24,16 @@
 
 package com.tencent.bk.job.common.service.feature;
 
+import com.tencent.bk.job.common.service.feature.config.FeatureToggleProperties;
 import com.tencent.bk.job.common.util.feature.FeatureManager;
 import com.tencent.bk.job.common.util.feature.FeatureStore;
 import io.micrometer.core.instrument.MeterRegistry;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
 @Configuration(proxyBeanMethods = false)
+@EnableConfigurationProperties({FeatureToggleProperties.class})
 public class FeatureToggleAutoConfiguration {
 
     @Bean

diff --git a/...service/src/main/java/com/tencent/bk/job/common/service/feature/InMemoryFeatureStore.java b/...service/src/main/java/com/tencent/bk/job/common/service/feature/InMemoryFeatureStore.java
@@ -25,7 +25,7 @@
 package com.tencent.bk.job.common.service.feature;
 
 import com.tencent.bk.job.common.service.feature.config.FeatureConfig;
-import com.tencent.bk.job.common.service.feature.config.FeatureToggleConfig;
+import com.tencent.bk.job.common.service.feature.config.FeatureToggleProperties;
 import com.tencent.bk.job.common.service.feature.config.ToggleStrategyConfig;
 import com.tencent.bk.job.common.service.feature.strategy.AllMatchToggleStrategy;
 import com.tencent.bk.job.common.service.feature.strategy.AnyMatchToggleStrategy;
@@ -95,17 +95,17 @@ public boolean load(boolean ignoreException) {
     private void loadInternal() {
         synchronized (this) {
             log.info("Load feature toggle start ...");
-            FeatureToggleConfig featureToggleConfig = ApplicationContextRegister.getBean(FeatureToggleConfig.class);
+            FeatureToggleProperties featureToggleProperties = ApplicationContextRegister.getBean(FeatureToggleProperties.class);
 
-            if (featureToggleConfig.getFeatures() == null || featureToggleConfig.getFeatures().isEmpty()) {
+            if (featureToggleProperties.getFeatures() == null || featureToggleProperties.getFeatures().isEmpty()) {
                 log.info("Feature toggle config empty!");
                 return;
             }
 
-            log.info("Parse feature toggle config: {}", JsonUtils.toJson(featureToggleConfig));
+            log.info("Parse feature toggle config: {}", JsonUtils.toJson(featureToggleProperties));
 
             Map<String, Feature> tmpFeatures = new HashMap<>();
-            featureToggleConfig.getFeatures().forEach((featureId, featureConfig) -> {
+            featureToggleProperties.getFeatures().forEach((featureId, featureConfig) -> {
                 Feature feature = parseFeatureConfig(featureId, featureConfig);
                 tmpFeatures.put(featureId, feature);
             });

diff --git a/...e/feature/config/FeatureToggleConfig.java → ...ature/config/FeatureToggleProperties.java b/...e/feature/config/FeatureToggleConfig.java → ...ature/config/FeatureToggleProperties.java
@@ -45,7 +45,7 @@
 @Getter
 @Setter
 @Slf4j
-public class FeatureToggleConfig {
+public class FeatureToggleProperties {
 
     /**
      * 特性
@@ -54,6 +54,6 @@ public class FeatureToggleConfig {
 
     @PostConstruct
     public void print() {
-        log.info("FeatureToggleConfig init: {}", JsonUtils.toJson(this));
+        log.info("FeatureToggleProperties init: {}", JsonUtils.toJson(this));
     }
 }
diff --git a/...n/java/com/tencent/bk/job/common/service/quota/config/ResourceQuotaAutoConfiguration.java b/...n/java/com/tencent/bk/job/common/service/quota/config/ResourceQuotaAutoConfiguration.java
@@ -27,13 +27,15 @@
 import com.tencent.bk.job.common.service.quota.ResourceQuotaLoadApplicationRunner;
 import com.tencent.bk.job.common.service.quota.ResourceQuotaStore;
 import com.tencent.bk.job.common.service.quota.RunningJobResourceQuotaStore;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
 /**
  * 配额限制 Spring AutoConfiguration
  */
 @Configuration(proxyBeanMethods = false)
+@EnableConfigurationProperties({ResourceQuotaLimitProperties.class})
 public class ResourceQuotaAutoConfiguration {
 
     @Bean

diff --git a/.../tencent/bk/job/common/service/quota/config/parser/AbstractResourceQuotaConfigParser.java b/.../tencent/bk/job/common/service/quota/config/parser/AbstractResourceQuotaConfigParser.java
@@ -50,7 +50,7 @@ protected ResourceScopeQuotaLimit parseResourceScopeQuotaLimit(Long capacity,
         );
 
         parseGlobalLimit(resourceScopeQuotaLimit, capacity, globalLimitExpr);
-        resourceScopeQuotaLimit.setCustomLimits(parseCustomLimit(capacity, customLimitExpr, key -> key));
+        parseCustomLimit(resourceScopeQuotaLimit, capacity, customLimitExpr);
 
         return resourceScopeQuotaLimit;
     }
@@ -64,7 +64,7 @@ protected AppQuotaLimit parseAppQuotaLimit(Long capacity,
         );
 
         parseGlobalLimit(appQuotaLimit, capacity, globalLimitExpr);
-        appQuotaLimit.setCustomLimits(parseCustomLimit(capacity, customLimitExpr, key -> key));
+        parseCustomLimit(appQuotaLimit, capacity, customLimitExpr);
 
         return appQuotaLimit;
     }
@@ -74,6 +74,14 @@ private void parseGlobalLimit(QuotaLimit quotaLimit, Long capacity, String globa
         quotaLimit.setGlobalLimit(globalLimit);
     }
 
+    private void parseCustomLimit(ResourceScopeQuotaLimit quotaLimit, Long capacity, String customLimitExpr) {
+        quotaLimit.setCustomLimits(parseCustomLimit(capacity, customLimitExpr, key -> key));
+    }
+
+    private void parseCustomLimit(AppQuotaLimit quotaLimit, Long capacity, String customLimitExpr) {
+        quotaLimit.setCustomLimits(parseCustomLimit(capacity, customLimitExpr, key -> key));
+    }
+
     protected <K> Map<K, Long> parseCustomLimit(Long capacity,
                                                 String customLimitExpr,
                                                 Function<String, K> customKeyParser) {

diff --git a/src/backend/commons/common-service/src/test/java/FeatureToggleTest.java b/src/backend/commons/common-service/src/test/java/FeatureToggleTest.java
@@ -26,7 +26,7 @@
 import com.tencent.bk.job.common.model.dto.ResourceScope;
 import com.tencent.bk.job.common.service.feature.DefaultFeatureManager;
 import com.tencent.bk.job.common.service.feature.InMemoryFeatureStore;
-import com.tencent.bk.job.common.service.feature.config.FeatureToggleConfig;
+import com.tencent.bk.job.common.service.feature.config.FeatureToggleProperties;
 import com.tencent.bk.job.common.service.feature.strategy.JobInstanceAttrToggleStrategy;
 import com.tencent.bk.job.common.util.ApplicationContextRegister;
 import com.tencent.bk.job.common.util.feature.FeatureExecutionContext;
@@ -54,14 +54,14 @@ class FeatureToggleTest {
 
     @BeforeAll
     static void beforeAll() {
-        Yaml yaml = new Yaml(new Constructor(FeatureToggleConfig.class));
+        Yaml yaml = new Yaml(new Constructor(FeatureToggleProperties.class));
         InputStream inputStream = FeatureToggleTest.class.getClassLoader()
             .getResourceAsStream("features_1.yaml");
-        FeatureToggleConfig featureToggleConfig = yaml.load(inputStream);
+        FeatureToggleProperties featureToggleProperties = yaml.load(inputStream);
 
         MockedStatic<ApplicationContextRegister> mockedStatic = Mockito.mockStatic(ApplicationContextRegister.class);
-        mockedStatic.when(() -> ApplicationContextRegister.getBean(FeatureToggleConfig.class))
-            .thenReturn(featureToggleConfig);
+        mockedStatic.when(() -> ApplicationContextRegister.getBean(FeatureToggleProperties.class))
+            .thenReturn(featureToggleProperties);
         FeatureStore mockFeatureStore = new InMemoryFeatureStore();
         mockFeatureStore.load(false);
         mockedStatic.close();

diff --git a/...ot-job-analysis/src/main/java/com/tencent/bk/job/analysis/JobAnalysisBootApplication.java b/...ot-job-analysis/src/main/java/com/tencent/bk/job/analysis/JobAnalysisBootApplication.java
@@ -25,12 +25,9 @@
 package com.tencent.bk.job.analysis;
 
 import com.tencent.bk.job.common.service.boot.JobBootApplication;
-import com.tencent.bk.job.common.service.feature.config.FeatureToggleConfig;
-import com.tencent.bk.job.common.service.quota.config.ResourceQuotaLimitProperties;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.scheduling.annotation.EnableScheduling;
@@ -47,7 +44,6 @@
     }
 )
 @EnableScheduling
-@EnableConfigurationProperties({FeatureToggleConfig.class, ResourceQuotaLimitProperties.class})
 public class JobAnalysisBootApplication {
 
     public static void main(String[] args) {

diff --git a/...nd/job-assemble/src/main/java/com/tencent/bk/job/assemble/JobAssembleBootApplication.java b/...nd/job-assemble/src/main/java/com/tencent/bk/job/assemble/JobAssembleBootApplication.java
@@ -25,12 +25,9 @@
 package com.tencent.bk.job.assemble;
 
 import com.tencent.bk.job.common.service.boot.JobBootApplication;
-import com.tencent.bk.job.common.service.feature.config.FeatureToggleConfig;
-import com.tencent.bk.job.common.service.quota.config.ResourceQuotaLimitProperties;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.scheduling.annotation.EnableScheduling;
@@ -52,7 +49,6 @@
 )
 @EnableCaching
 @EnableScheduling
-@EnableConfigurationProperties({FeatureToggleConfig.class, ResourceQuotaLimitProperties.class})
 @EnableFeignClients(basePackages = "com.tencent.bk.job")
 public class JobAssembleBootApplication {
 

diff --git a/...kup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/JobBackupBootApplication.java b/...kup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/JobBackupBootApplication.java
@@ -25,12 +25,9 @@
 package com.tencent.bk.job.backup;
 
 import com.tencent.bk.job.common.service.boot.JobBootApplication;
-import com.tencent.bk.job.common.service.feature.config.FeatureToggleConfig;
-import com.tencent.bk.job.common.service.quota.config.ResourceQuotaLimitProperties;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.scheduling.annotation.EnableScheduling;
@@ -46,7 +43,6 @@
     }
 )
 @EnableScheduling
-@EnableConfigurationProperties({FeatureToggleConfig.class, ResourceQuotaLimitProperties.class})
 public class JobBackupBootApplication {
     public static void main(String[] args) {
         SpringApplication.run(JobBackupBootApplication.class, args);

diff --git a/.../boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/JobCrontabBootApplication.java b/.../boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/JobCrontabBootApplication.java
@@ -25,20 +25,16 @@
 package com.tencent.bk.job.crontab;
 
 import com.tencent.bk.job.common.service.boot.JobBootApplication;
-import com.tencent.bk.job.common.service.feature.config.FeatureToggleConfig;
-import com.tencent.bk.job.common.service.quota.config.ResourceQuotaLimitProperties;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 
 @JobBootApplication(
     scanBasePackages = {
         "com.tencent.bk.job.crontab"},
     exclude = {JooqAutoConfiguration.class, ApplicationAvailabilityAutoConfiguration.class})
 @EnableFeignClients(basePackages = {"com.tencent.bk.job.manage.api", "com.tencent.bk.job.execute.api"})
-@EnableConfigurationProperties({FeatureToggleConfig.class, ResourceQuotaLimitProperties.class})
 public class JobCrontabBootApplication {
 
     public static void main(String[] args) {

diff --git a/.../boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java b/.../boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java
@@ -25,12 +25,9 @@
 package com.tencent.bk.job.execute;
 
 import com.tencent.bk.job.common.service.boot.JobBootApplication;
-import com.tencent.bk.job.common.service.feature.config.FeatureToggleConfig;
-import com.tencent.bk.job.common.service.quota.config.ResourceQuotaLimitProperties;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 
 @JobBootApplication(
@@ -43,7 +40,6 @@
         "com.tencent.bk.job.file_gateway.api"
     }
 )
-@EnableConfigurationProperties({FeatureToggleConfig.class, ResourceQuotaLimitProperties.class})
 public class JobExecuteBootApplication {
 
     public static void main(String[] args) {

diff --git a/...ute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java b/...ute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java
@@ -166,7 +166,7 @@ public Response<TaskExecuteVO> redoTask(String username,
         }
 
         List<TaskVariableDTO> executeVariableValues = buildExecuteVariables(request.getTaskVariables());
-        TaskInstanceDTO taskInstanceDTO = taskExecuteService.createTaskInstanceForRedo(appResourceScope.getAppId(),
+        TaskInstanceDTO taskInstanceDTO = taskExecuteService.redoJob(appResourceScope.getAppId(),
             request.getTaskInstanceId(), username, executeVariableValues);
 
         TaskExecuteVO result = new TaskExecuteVO();

diff --git a/...in/java/com/tencent/bk/job/execute/engine/quota/limit/RunningJobResourceQuotaManager.java b/...in/java/com/tencent/bk/job/execute/engine/quota/limit/RunningJobResourceQuotaManager.java
@@ -53,7 +53,7 @@
 import java.util.stream.Collectors;
 
 /**
- * 正在执行作业配置限制管理
+ * 正在执行作业配额限制管理
  */
 @Component
 @Slf4j

diff --git a/...vice-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java b/...vice-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java
@@ -70,17 +70,17 @@ public interface TaskExecuteService {
     TaskInstanceDTO executeJobPlan(TaskExecuteParam executeParam);
 
     /**
-     * 创建重做作业实例
+     * 重做作业
      *
      * @param appId                 业务 ID
      * @param taskInstanceId        作业实例 ID
      * @param operator              操作者
      * @param executeVariableValues 全局变量
-     * @return
      */
-    TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, String operator,
-                                              List<TaskVariableDTO> executeVariableValues)
-        throws ServiceException;
+    TaskInstanceDTO redoJob(Long appId,
+                            Long taskInstanceId,
+                            String operator,
+                            List<TaskVariableDTO> executeVariableValues);
 
     /**
      * 步骤操作
@@ -91,7 +91,7 @@ TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, Strin
      * @return 执行次数
      * @throws ServiceException
      */
-    Integer doStepOperation(Long appId, String operator, StepOperationDTO stepOperation) throws ServiceException;
+    Integer doStepOperation(Long appId, String operator, StepOperationDTO stepOperation);
 
     /**
      * 终止作业
@@ -101,7 +101,7 @@ TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, Strin
      * @param taskInstanceId 作业实例ID
      * @throws ServiceException 终止操作异常的时候抛出
      */
-    void terminateJob(String username, Long appId, Long taskInstanceId) throws ServiceException;
+    void terminateJob(String username, Long appId, Long taskInstanceId);
 
     /**
      * 作业操作
@@ -110,15 +110,16 @@ TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, Strin
      * @param operator       操作者
      * @param taskInstanceId 作业实例 ID
      * @param operation      操作类型
-     * @throws ServiceException
      */
-    void doTaskOperation(Long appId, String operator, long taskInstanceId,
-                         TaskOperationEnum operation) throws ServiceException;
+    void doTaskOperation(Long appId,
+                         String operator,
+                         long taskInstanceId,
+                         TaskOperationEnum operation);
 
     /**
      * 作业执行方案执行鉴权
      *
      * @param executeParam 作业执行参数
      */
-    void authExecuteJobPlan(TaskExecuteParam executeParam) throws ServiceException;
+    void authExecuteJobPlan(TaskExecuteParam executeParam);
 }