perf: 容器化Values暴露定时任务独立DB等生产环境需要的配置项 TencentBlueKing#2551

暴露相关配置项
jsonwan · Nov 6, 2023 · ae3c707 · ae3c707
1 parent a451d4a
commit ae3c707
Show file tree

Hide file tree

Showing 14 changed files with 169 additions and 248 deletions.
diff --git a/.../boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/JobCrontabBootApplication.java b/.../boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/JobCrontabBootApplication.java
@@ -26,7 +26,6 @@
 
 import com.tencent.bk.job.common.service.boot.JobBootApplication;
 import com.tencent.bk.job.common.service.feature.config.FeatureToggleConfig;
-import com.tencent.bk.job.crontab.config.JobQuartzProperties;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration;
@@ -38,7 +37,7 @@
         "com.tencent.bk.job.crontab"},
     exclude = {JooqAutoConfiguration.class, ApplicationAvailabilityAutoConfiguration.class})
 @EnableFeignClients(basePackages = {"com.tencent.bk.job.manage.api", "com.tencent.bk.job.execute.api"})
-@EnableConfigurationProperties({FeatureToggleConfig.class, JobQuartzProperties.class})
+@EnableConfigurationProperties({FeatureToggleConfig.class})
 public class JobCrontabBootApplication {
 
     public static void main(String[] args) {

diff --git a/...vice-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/JobQuartzProperties.java b/...vice-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/JobQuartzProperties.java
diff --git a/...tab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/QuartzConfig.java b/...tab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/QuartzConfig.java
@@ -24,14 +24,11 @@
 
 package com.tencent.bk.job.crontab.config;
 
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.autoconfigure.quartz.SchedulerFactoryBeanCustomizer;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Profile;
-import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 
 import javax.sql.DataSource;
 
@@ -40,54 +37,15 @@
  */
 @Configuration
 @Profile("!test")
-@EnableConfigurationProperties({JobQuartzProperties.class})
 public class QuartzConfig {
 
-    private final JobQuartzProperties jobQuartzProperties;
-
-    @Autowired
-    public QuartzConfig(JobQuartzProperties quartzProperties) {
-        this.jobQuartzProperties = quartzProperties;
-    }
-
-    @Bean("quartzTaskExecutor")
-    public ThreadPoolTaskExecutor quartzTaskExecutor() {
-        JobQuartzProperties.ThreadPool threadPoolConfig = jobQuartzProperties.getThreadPool();
-        ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor();
-        threadPoolTaskExecutor.setThreadNamePrefix(threadPoolConfig.getThreadNamePrefix());
-        threadPoolTaskExecutor.setThreadGroupName(threadPoolConfig.getThreadGroupName());
-        threadPoolTaskExecutor.setDaemon(threadPoolConfig.isDaemon());
-        threadPoolTaskExecutor.setThreadPriority(threadPoolConfig.getThreadPriority());
-        threadPoolTaskExecutor.setCorePoolSize(threadPoolConfig.getCorePoolSize());
-        threadPoolTaskExecutor.setMaxPoolSize(threadPoolConfig.getMaxPoolSize());
-        threadPoolTaskExecutor.setQueueCapacity(threadPoolConfig.getQueueCapacity());
-        threadPoolTaskExecutor.setKeepAliveSeconds(threadPoolConfig.getKeepAliveSeconds());
-        threadPoolTaskExecutor.setAllowCoreThreadTimeOut(threadPoolConfig.isAllowCoreThreadTimeOut());
-        threadPoolTaskExecutor.setWaitForTasksToCompleteOnShutdown(
-            threadPoolConfig.isWaitForTasksToCompleteOnShutdown());
-        threadPoolTaskExecutor.setAwaitTerminationSeconds(
-            threadPoolConfig.getAwaitTerminationSeconds());
-
-        return threadPoolTaskExecutor;
-    }
-
     @Bean
     public SchedulerFactoryBeanCustomizer schedulerFactoryBeanCustomizer(
-        @Qualifier("quartzTaskExecutor") ThreadPoolTaskExecutor quartzTaskExecutor,
         @Qualifier("job-crontab-data-source") DataSource dataSource) {
         return schedulerFactoryBean -> {
-            // 自定义taskExecutor
-            schedulerFactoryBean.setTaskExecutor(quartzTaskExecutor);
-
-            // 自定义scheduler
-            schedulerFactoryBean.setSchedulerName(jobQuartzProperties.getScheduler().getSchedulerName());
-            schedulerFactoryBean.setApplicationContextSchedulerContextKey(
-                jobQuartzProperties.getScheduler().getApplicationContextSchedulerContextKey());
-            schedulerFactoryBean.setOverwriteExistingJobs(jobQuartzProperties.getScheduler().isOverwriteExistingJobs());
-            schedulerFactoryBean.setAutoStartup(jobQuartzProperties.getScheduler().isAutoStartup());
-            schedulerFactoryBean.setStartupDelay(jobQuartzProperties.getScheduler().getStartupDelay());
+            schedulerFactoryBean.setOverwriteExistingJobs(true);
+            schedulerFactoryBean.setStartupDelay(10);
             schedulerFactoryBean.setDataSource(dataSource);
         };
-
     }
 }
diff --git a/support-files/kubernetes/charts/bk-job/VALUES_LOG.md b/support-files/kubernetes/charts/bk-job/VALUES_LOG.md
@@ -1,5 +1,21 @@
 # chart values 更新日志
 
+## 0.5.4
+1. 增加定时任务服务独立数据库配置（若不配置该项则使用与其他服务共用的公共数据库），默认无需配置
+
+```yaml
+## job-crontab定时任务配置
+crontabConfig:
+  # 定时任务服务独立数据库配置，若不配置该项则使用与其他服务共用的公共数据库
+  database:
+    host: ""
+    port: 3306
+    connection:
+      properties: ?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull
+    username: "job"
+    password: "job"
+```
+
 ## 0.5.3
 1. 增加操作审计相关配置
 

diff --git a/support-files/kubernetes/charts/bk-job/templates/_helpers.tpl b/support-files/kubernetes/charts/bk-job/templates/_helpers.tpl
@@ -669,7 +669,6 @@ Return the Archive MariaDB secret name
 {{ printf "%s-%s" (include "job.fullname" .) "archive-mariadb" }}
 {{- end -}}
 
-
 {{/*
 Return the storage PVC name
 */}}
@@ -683,3 +682,30 @@ Return the job pod terminationGracePeriodSeconds
 {{- define "job.podTerminationGracePeriodSeconds" -}}
 terminationGracePeriodSeconds: {{ .Values.podTerminationGracePeriodSeconds }}
 {{- end -}}
+
+{{/*
+Return the Crontab DB secret name
+*/}}
+{{- define "job.crontabDB.secretName" -}}
+{{ printf "%s-%s" (include "job.fullname" .) "crontab-db" }}
+{{- end -}}
+
+
+{{/*
+Return the job crontab database config
+*/}}
+{{- define "job.crontab.databaseConfig" -}}
+{{- if .Values.crontabConfig.database.host -}}
+jdbc-url: {{ include "job.jdbcMysqlScheme" . }}://{{- .Values.crontabConfig.database.host }}:{{- .Values.crontabConfig.database.port }}/job_crontab{{- .Values.crontabConfig.database.connection.properties }}
+username: {{ .Values.crontabConfig.database.username }}
+password: ${crontab-db-password}
+{{- else -}}
+jdbc-url: {{ include "job.jdbcMysqlScheme" . }}://{{- include "job.mariadb.host" . }}:{{- include "job.mariadb.port" . }}/job_crontab{{ include "job.mariadb.connection.properties" . }}
+username: {{ include "job.mariadb.username" . }}
+    {{- if .Values.externalMariaDB.existingPasswordSecret }}
+password: {{ .Values.externalMariaDB.existingPasswordKey | default "mariadb-password" | printf "${%s}" }}
+    {{- else }}
+password: ${mariadb-password}
+    {{- end }}
+{{- end }}
+{{- end -}}
diff --git a/support-files/kubernetes/charts/bk-job/templates/job-analysis/configmap.yaml b/support-files/kubernetes/charts/bk-job/templates/job-analysis/configmap.yaml
@@ -62,11 +62,7 @@ data:
           max-file-size: 5GB
           max-request-size: 5GB
     feign:
-      client:
-        config:
-          default:
-            connectTimeout: 5000
-            readTimeout: 300000
+      {{- toYaml .Values.analysisConfig.feign | nindent 6 }}
     server:
       port: {{ .Values.analysisConfig.containerPort }}
 {{- end }}
diff --git a/support-files/kubernetes/charts/bk-job/templates/job-assemble/configmap.yaml b/support-files/kubernetes/charts/bk-job/templates/job-assemble/configmap.yaml
@@ -188,13 +188,7 @@ data:
         job-crontab:
           driver-class-name: {{ include "job.jdbcMysqlDriverClass" . }}
           type: com.zaxxer.hikari.HikariDataSource
-          jdbc-url: {{ include "job.jdbcMysqlScheme" . }}://{{- include "job.mariadb.host" . }}:{{- include "job.mariadb.port" . }}/job_crontab{{ include "job.mariadb.connection.properties" . }}
-          username: {{ include "job.mariadb.username" . }}
-          {{ if .Values.externalMariaDB.existingPasswordSecret }}
-          password: {{ .Values.externalMariaDB.existingPasswordKey | default "mariadb-password" | printf "${%s}" }}
-          {{- else -}}
-          password: ${mariadb-password}
-          {{- end }}
+          {{- include "job.crontab.databaseConfig" . | nindent 10 }}
           maximum-pool-size: 100
           minimum-idle: 20
           idle-timeout: 600000
@@ -278,24 +272,6 @@ data:
         jdbc:
           # 是否自动使用 SQL 初始化 Quartz 表结构。这里设置成 never ，手动创建表结构
           initialize-schema: never
-        thread-pool:
-          thread-name-prefix: QuartzThreadPoolTaskExecutor-
-          thread-priority: 5
-          daemon: false
-          thread-group-name: QuartzThreadPoolTaskExecutorGroup
-          core-pool-size: 150
-          max-pool-size: 300
-          keep-alive-seconds: 60
-          queue-capacity: 150
-          allow-core-thread-timeout: false
-          waitfor-tasks-tocomplete-onshutdown: true
-          await-termination-seconds: 900
-        scheduler:
-          scheduler-name: quartz-scheduler
-          application-context-scheduler-contextkey: applicationContext
-          overwrite-existing-jobs: true
-          auto-startup: true
-          startup-delay: 10
         properties:
           org:
             quartz:
@@ -319,20 +295,16 @@ data:
                 # 调度器实例编号自动生成
                 instanceId: AUTO
                 skipUpdateCheck: true
-                batchTriggerAcquisitionMaxCount: 50
+                batchTriggerAcquisitionMaxCount: {{ .Values.crontabConfig.quartz.scheduler.batchTriggerAcquisitionMaxCount }}
               threadPool:
                 class: org.quartz.simpl.SimpleThreadPool
-                threadCount: 300
+                threadCount: {{ .Values.crontabConfig.quartz.threadPool.threadCount }}
 
     ## 其他框架组件配置 ##
     server:
       port: {{ .Values.assembleConfig.containerPort }}
     feign:
-      client:
-        config:
-          default:
-            connectTimeout: 5000
-            readTimeout: 20000
+      {{- toYaml .Values.assembleConfig.feign | nindent 6 }}
 
     ## Job 配置 ##
     job:

diff --git a/support-files/kubernetes/charts/bk-job/templates/job-assemble/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-assemble/deployment.yaml
@@ -123,6 +123,11 @@ spec:
           resources:
             {{- toYaml .Values.assembleConfig.resources | nindent 12 }}
           volumeMounts:
+            {{- if .Values.crontabConfig.database.host }}
+            - name: crontabDB
+              mountPath: /etc/secrets/crontabDB
+              readOnly: true
+            {{- end }}
             - name: distribute-volume
               mountPath: {{ .Values.fileDistribute.hostPath }}
             - name: job-storage
@@ -144,6 +149,11 @@ spec:
               readOnly: true
       terminationGracePeriodSeconds: 120
       volumes:
+        {{- if .Values.crontabConfig.database.host }}
+        - name: crontabDB
+          secret:
+            secretName: {{ include "job.crontabDB.secretName" . }}
+        {{- end }}
         - name: distribute-volume
           hostPath:
             path: {{ .Values.fileDistribute.hostPath }}