Merge pull request #3075 from wangyu096/issue_261

feat: 执行引擎任务调度配额限制 #261

src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties

@@ -181,6 +181,9 @@
 1244029=步骤 [{0}] 的源文件执行对象为空
 1244030=作业引用的执行对象不存在。不存在的执行对象个数:{0},执行对象列表[{1}]
 1244031=Label Selector 不合法
+1244032=当前执行的作业总量超过业务配额限制
+1244033=当前执行的作业总量超过应用配额限制
+1244034=当前执行的作业总量超过系统配额限制
 
 ## 业务错误-定时任务(job-crontab)
 1245008=删除定时任务失败

src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties

@@ -181,6 +181,9 @@
 1244029=Step [{0}] source execute object is empty
 1244030=Execute object referenced by the job does not exist. Number of non-existent execution objects: {0}, execution object list: [{1}]
 1244031=Invalid Label Selector
+1244032=The total number of jobs currently executed exceeds the resource scope quota limit
+1244033=The total number of jobs currently executed exceeds the bk app quota limit
+1244034=The total number of jobs currently executed exceeds the system quota limit
 
 ## Business error - job-crontab
 1245008=Failed to delete cron

src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties

@@ -181,6 +181,9 @@
 1244029=Step [{0}] source execute object is empty
 1244030=Execute object referenced by the job does not exist. Number of non-existent execution objects: {0}, execution object list: [{1}]
 1244031=Invalid Label Selector
+1244032=The total number of jobs currently executed exceeds the resource scope quota limit
+1244033=The total number of jobs currently executed exceeds the bk app quota limit
+1244034=The total number of jobs currently executed exceeds the system quota limit
 
 ## Business error - job-crontab
 1245008=Failed to delete cron

src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties

@@ -181,6 +181,9 @@
 1244029=步骤 [{0}] 的源文件执行对象为空
 1244030=作业引用的执行对象不存在。不存在的执行对象个数:{0},执行对象列表[{1}]
 1244031=Label Selector 不合法
+1244032=当前执行的作业总量超过业务配额限制
+1244033=当前执行的作业总量超过应用配额限制
+1244034=当前执行的作业总量超过系统配额限制
 
 ## 业务错误-定时任务(job-crontab)
 1245008=删除定时任务失败

src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties

@@ -182,6 +182,9 @@
 1244029=步骤 [{0}] 的源文件执行对象为空
 1244030=作业引用的执行对象不存在。不存在的执行对象个数:{0},执行对象列表[{1}]
 1244031=Label Selector 不合法
+1244032=当前执行的作业总量超过业务配额限制
+1244033=当前执行的作业总量超过应用配额限制
+1244034=当前执行的作业总量超过系统配额限制
 
 ## 业务错误-定时任务(job-crontab)
 1245008=删除定时任务失败

src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/CommonServiceAutoConfiguration.java

@@ -24,16 +24,20 @@
 
 package com.tencent.bk.job.common.service;
 
+import com.tencent.bk.job.common.service.quota.ResourceQuotaStore;
 import com.tencent.bk.job.common.util.feature.FeatureStore;
+import io.micrometer.core.instrument.MeterRegistry;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
 @Configuration(proxyBeanMethods = false)
 public class CommonServiceAutoConfiguration {
 
     @Bean
-    public ConfigRefreshEventListener configRefreshEventListener(FeatureStore featureStore) {
-        return new ConfigRefreshEventListener(featureStore);
+    public ConfigRefreshEventListener configRefreshEventListener(MeterRegistry meterRegistry,
+                                                                 FeatureStore featureStore,
+                                                                 ResourceQuotaStore resourceQuotaStore) {
+        return new ConfigRefreshEventListener(meterRegistry, featureStore, resourceQuotaStore);
     }
 
     @Bean

src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/ConfigRefreshEventListener.java

@@ -24,7 +24,10 @@
 
 package com.tencent.bk.job.common.service;
 
+import com.tencent.bk.job.common.service.quota.ResourceQuotaStore;
 import com.tencent.bk.job.common.util.feature.FeatureStore;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tags;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.cloud.context.environment.EnvironmentChangeEvent;
@@ -39,10 +42,22 @@
 @Slf4j
 public class ConfigRefreshEventListener {
 
+    private final MeterRegistry meterRegistry;
+
     private final FeatureStore featureStore;
 
-    public ConfigRefreshEventListener(FeatureStore featureStore) {
+    private final ResourceQuotaStore resourceQuotaStore;
+
+    private static final String METRIC_JOB_CONFIG_REFRESH_FAIL_TOTAL = "job_config_refresh_fail_total";
+    private static final String METRIC_TAG_CONFIG_NAME = "config_name";
+
+
+    public ConfigRefreshEventListener(MeterRegistry meterRegistry,
+                                      FeatureStore featureStore,
+                                      ResourceQuotaStore resourceQuotaStore) {
+        this.meterRegistry = meterRegistry;
         this.featureStore = featureStore;
+        this.resourceQuotaStore = resourceQuotaStore;
         log.info("Init ConfigRefreshEventListener");
     }
 
@@ -56,7 +71,7 @@ public void onEvent(EnvironmentChangeEvent event) {
         if (log.isInfoEnabled()) {
             log.info("Handle EnvironmentChangeEvent, event: {}", printEnvironmentChangeEvent(event));
         }
-        reloadFeatureToggleIfChanged(event.getKeys());
+        reload(event.getKeys());
     }
 
     private String printEnvironmentChangeEvent(EnvironmentChangeEvent event) {
@@ -67,18 +82,31 @@ private String printEnvironmentChangeEvent(EnvironmentChangeEvent event) {
             .toString();
     }
 
-
     /**
-     * 重载特性开关配置
+     * 重载配置
      */
-    private void reloadFeatureToggleIfChanged(Set<String> changedKeys) {
+    private void reload(Set<String> changedKeys) {
         if (CollectionUtils.isEmpty(changedKeys)) {
             return;
         }
-        boolean isFeatureToggleConfigChanged =
-            changedKeys.stream().anyMatch(changedKey -> changedKey.startsWith("job.features."));
-        if (isFeatureToggleConfigChanged) {
-            featureStore.load(true);
+        if (changedKeys.stream().anyMatch(changedKey -> changedKey.startsWith("job.features."))) {
+            boolean handleResult = featureStore.handleConfigChange();
+            if (!handleResult) {
+                meterRegistry.counter(
+                        METRIC_JOB_CONFIG_REFRESH_FAIL_TOTAL,
+                        Tags.of(METRIC_TAG_CONFIG_NAME, "job.feature"))
+                    .increment();
+            }
+        }
+
+        if (changedKeys.stream().anyMatch(changedKey -> changedKey.startsWith("job.resourceQuotaLimit."))) {
+            boolean handleResult = resourceQuotaStore.handleConfigChange();
+            if (!handleResult) {
+                meterRegistry.counter(
+                        METRIC_JOB_CONFIG_REFRESH_FAIL_TOTAL,
+                        Tags.of(METRIC_TAG_CONFIG_NAME, "job.resourceQuotaLimit"))
+                    .increment();
+            }
         }
     }
 }

src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feature/FeatureToggleAutoConfiguration.java

@@ -24,13 +24,16 @@
 
 package com.tencent.bk.job.common.service.feature;
 
+import com.tencent.bk.job.common.service.feature.config.FeatureToggleProperties;
 import com.tencent.bk.job.common.util.feature.FeatureManager;
 import com.tencent.bk.job.common.util.feature.FeatureStore;
 import io.micrometer.core.instrument.MeterRegistry;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
 @Configuration(proxyBeanMethods = false)
+@EnableConfigurationProperties({FeatureToggleProperties.class})
 public class FeatureToggleAutoConfiguration {
 
     @Bean

src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feature/InMemoryFeatureStore.java

@@ -25,7 +25,7 @@
 package com.tencent.bk.job.common.service.feature;
 
 import com.tencent.bk.job.common.service.feature.config.FeatureConfig;
-import com.tencent.bk.job.common.service.feature.config.FeatureToggleConfig;
+import com.tencent.bk.job.common.service.feature.config.FeatureToggleProperties;
 import com.tencent.bk.job.common.service.feature.config.ToggleStrategyConfig;
 import com.tencent.bk.job.common.service.feature.strategy.AllMatchToggleStrategy;
 import com.tencent.bk.job.common.service.feature.strategy.AnyMatchToggleStrategy;
@@ -76,33 +76,37 @@ public Feature getFeature(String featureId) {
     }
 
     @Override
-    public void load(boolean ignoreException) {
+    public boolean load(boolean ignoreException) {
+        boolean loadResult = true;
         try {
             loadInternal();
         } catch (Throwable e) {
             log.warn("Load feature config error", e);
+            loadResult = false;
             if (ignoreException) {
                 log.warn("Ignore feature config load error");
             } else {
                 throw e;
             }
         }
+        return loadResult;
     }
 
     private void loadInternal() {
         synchronized (this) {
             log.info("Load feature toggle start ...");
-            FeatureToggleConfig featureToggleConfig = ApplicationContextRegister.getBean(FeatureToggleConfig.class);
+            FeatureToggleProperties featureToggleProperties =
+                ApplicationContextRegister.getBean(FeatureToggleProperties.class);
 
-            if (featureToggleConfig.getFeatures() == null || featureToggleConfig.getFeatures().isEmpty()) {
+            if (featureToggleProperties.getFeatures() == null || featureToggleProperties.getFeatures().isEmpty()) {
                 log.info("Feature toggle config empty!");
                 return;
             }
 
-            log.info("Parse feature toggle config: {}", JsonUtils.toJson(featureToggleConfig));
+            log.info("Parse feature toggle config: {}", JsonUtils.toJson(featureToggleProperties));
 
             Map<String, Feature> tmpFeatures = new HashMap<>();
-            featureToggleConfig.getFeatures().forEach((featureId, featureConfig) -> {
+            featureToggleProperties.getFeatures().forEach((featureId, featureConfig) -> {
                 Feature feature = parseFeatureConfig(featureId, featureConfig);
                 tmpFeatures.put(featureId, feature);
             });

src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feature/config/FeatureToggleConfig.java → src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feature/config/FeatureToggleProperties.java

@@ -45,7 +45,7 @@
 @Getter
 @Setter
 @Slf4j
-public class FeatureToggleConfig {
+public class FeatureToggleProperties {
 
     /**
      * 特性
@@ -54,6 +54,6 @@ public class FeatureToggleConfig {
 
     @PostConstruct
     public void print() {
-        log.info("FeatureToggleConfig init: {}", JsonUtils.toJson(this));
+        log.info("FeatureToggleProperties init: {}", JsonUtils.toJson(this));
     }
 }

src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/ResourceQuotaConfigParseException.java

@@ -0,0 +1,45 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
+ *
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
+ *
+ * License for BK-JOB蓝鲸智云作业平台:
+ * --------------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+ * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+package com.tencent.bk.job.common.service.quota;
+
+/**
+ * 资源配额限制配置解析异常
+ */
+public class ResourceQuotaConfigParseException extends RuntimeException {
+    public ResourceQuotaConfigParseException() {
+    }
+
+    public ResourceQuotaConfigParseException(String message) {
+        super(message);
+    }
+
+    public ResourceQuotaConfigParseException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public ResourceQuotaConfigParseException(Throwable cause) {
+        super(cause);
+    }
+}

src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/ResourceQuotaLoadApplicationRunner.java

@@ -0,0 +1,49 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
+ *
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
+ *
+ * License for BK-JOB蓝鲸智云作业平台:
+ * --------------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+ * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+package com.tencent.bk.job.common.service.quota;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
+
+/**
+ * 配额限制配置加载 ApplicationRunner
+ */
+@Slf4j
+public class ResourceQuotaLoadApplicationRunner implements ApplicationRunner {
+    private final ResourceQuotaStore resourceQuotaStore;
+
+    public ResourceQuotaLoadApplicationRunner(ResourceQuotaStore resourceQuotaStore) {
+        this.resourceQuotaStore = resourceQuotaStore;
+    }
+
+    @Override
+    public void run(ApplicationArguments args) throws Exception {
+        // 初始化配置；如果初始化错误，那么抛出异常终止程序启动
+        log.info("ResourceQuotaLoadApplicationRunner start");
+        resourceQuotaStore.load(false);
+        log.info("ResourceQuotaLoadApplicationRunner run success");
+    }
+}