perf: 容器化环境支持无损更新 TencentBlueKing#2338

1. k8s环境下支持停止接收流量后延迟一段时间关闭Spring容器，用于无损更新/扩缩容； 2. 基础镜像增加网络调试相关工具。
jsonwan · Aug 17, 2023 · 9ca7772 · 9ca7772
1 parent 7424e6d
commit 9ca7772
Show file tree

Hide file tree

Showing 17 changed files with 91 additions and 14 deletions.
diff --git a/.../com/tencent/bk/job/common/consul/config/JobApplicationAvailabilityAutoConfiguration.java b/.../com/tencent/bk/job/common/consul/config/JobApplicationAvailabilityAutoConfiguration.java
@@ -0,0 +1,15 @@
+package com.tencent.bk.job.common.consul.config;
+
+import org.springframework.boot.availability.ApplicationAvailabilityBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration(proxyBeanMethods = false)
+public class JobApplicationAvailabilityAutoConfiguration {
+
+    @Bean
+    public ApplicationAvailabilityBean applicationAvailability() {
+        return new ApplicationAvailabilityBean();
+    }
+
+}
diff --git a/src/backend/commons/common-consul/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-consul/src/main/resources/META-INF/spring.factories
@@ -0,0 +1,2 @@
+org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
+com.tencent.bk.job.common.consul.config.JobApplicationAvailabilityAutoConfiguration
diff --git a/.../main/java/com/tencent/bk/job/common/k8s/availability/JobApplicationAvailabilityBean.java b/.../main/java/com/tencent/bk/job/common/k8s/availability/JobApplicationAvailabilityBean.java
@@ -0,0 +1,23 @@
+package com.tencent.bk.job.common.k8s.availability;
+
+import com.tencent.bk.job.common.util.ThreadUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.availability.ApplicationAvailabilityBean;
+import org.springframework.boot.availability.AvailabilityChangeEvent;
+import org.springframework.boot.availability.ReadinessState;
+
+@Slf4j
+public class JobApplicationAvailabilityBean extends ApplicationAvailabilityBean {
+    @Override
+    public void onApplicationEvent(AvailabilityChangeEvent<?> event) {
+        super.onApplicationEvent(event);
+        if (ReadinessState.REFUSING_TRAFFIC == event.getState()) {
+            // SpringCloud负载均衡缓存默认为35s，等待调用方缓存刷新后再真正关闭Spring容器
+            int waitSeconds = 35;
+            while (waitSeconds > 0) {
+                ThreadUtils.sleep(1000);
+                log.info("wait for GracefulShutdown, {}s left", waitSeconds--);
+            }
+        }
+    }
+}
diff --git a/...ava/com/tencent/bk/job/common/k8s/config/JobApplicationAvailabilityAutoConfiguration.java b/...ava/com/tencent/bk/job/common/k8s/config/JobApplicationAvailabilityAutoConfiguration.java
@@ -0,0 +1,16 @@
+package com.tencent.bk.job.common.k8s.config;
+
+import com.tencent.bk.job.common.k8s.availability.JobApplicationAvailabilityBean;
+import org.springframework.boot.availability.ApplicationAvailabilityBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration(proxyBeanMethods = false)
+public class JobApplicationAvailabilityAutoConfiguration {
+
+    @Bean
+    public ApplicationAvailabilityBean applicationAvailability() {
+        return new JobApplicationAvailabilityBean();
+    }
+
+}
diff --git a/src/backend/commons/common-k8s/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-k8s/src/main/resources/META-INF/spring.factories
@@ -1,2 +1,3 @@
 org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
-com.tencent.bk.job.common.k8s.config.K8sServiceInfoServiceAutoConfig
+com.tencent.bk.job.common.k8s.config.K8sServiceInfoServiceAutoConfig,\
+com.tencent.bk.job.common.k8s.config.JobApplicationAvailabilityAutoConfiguration
diff --git a/...ot-job-analysis/src/main/java/com/tencent/bk/job/analysis/JobAnalysisBootApplication.java b/...ot-job-analysis/src/main/java/com/tencent/bk/job/analysis/JobAnalysisBootApplication.java
@@ -27,14 +27,16 @@
 import com.tencent.bk.job.common.config.FeatureToggleConfig;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.DependsOn;
 import org.springframework.scheduling.annotation.EnableScheduling;
 
-@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class})
+@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class,
+    ApplicationAvailabilityAutoConfiguration.class})
 @EnableCaching
 @EnableFeignClients
 @EnableScheduling

diff --git a/...kup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/JobBackupBootApplication.java b/...kup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/JobBackupBootApplication.java
@@ -27,6 +27,7 @@
 import com.tencent.bk.job.common.config.FeatureToggleConfig;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cache.annotation.EnableCaching;
@@ -37,7 +38,8 @@
 /**
  * @since 21/7/2020 10:55
  */
-@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class})
+@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class,
+    ApplicationAvailabilityAutoConfiguration.class})
 @EnableCaching
 @EnableFeignClients
 @EnableScheduling

diff --git a/src/backend/job-config/src/main/java/com/tencent/bk/job/config/JobConfigBootApplication.java b/src/backend/job-config/src/main/java/com/tencent/bk/job/config/JobConfigBootApplication.java
@@ -26,12 +26,14 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.cloud.config.server.EnableConfigServer;
 
 /**
  * @date 2019/09/20
  */
-@SpringBootApplication(scanBasePackages = "com.tencent.bk.job")
+@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude =
+    {ApplicationAvailabilityAutoConfiguration.class})
 @EnableConfigServer
 public class JobConfigBootApplication {
 

diff --git a/.../boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/JobCrontabBootApplication.java b/.../boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/JobCrontabBootApplication.java
@@ -28,14 +28,16 @@
 import com.tencent.bk.job.crontab.config.JobQuartzProperties;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 
 /**
  * @date 2019/09/20
  */
-@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class})
+@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class,
+    ApplicationAvailabilityAutoConfiguration.class})
 @EnableFeignClients
 @EnableConfigurationProperties({FeatureToggleConfig.class, JobQuartzProperties.class})
 public class JobCrontabBootApplication {

diff --git a/.../boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java b/.../boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java
@@ -27,13 +27,15 @@
 import com.tencent.bk.job.common.config.FeatureToggleConfig;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 
 
-@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class})
+@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class,
+    ApplicationAvailabilityAutoConfiguration.class})
 @EnableDiscoveryClient
 @EnableFeignClients
 @EnableConfigurationProperties({FeatureToggleConfig.class})

diff --git a/...-gateway/src/main/java/com/tencent/bk/job/file_gateway/JobFileGatewayBootApplication.java b/...-gateway/src/main/java/com/tencent/bk/job/file_gateway/JobFileGatewayBootApplication.java
@@ -28,14 +28,16 @@
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.scheduling.annotation.EnableScheduling;
 
 @Slf4j
-@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class})
+@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class,
+    ApplicationAvailabilityAutoConfiguration.class})
 @EnableCaching
 @EnableFeignClients
 @EnableScheduling

diff --git a/...ckend/job-gateway/src/main/java/com/tencent/bk/job/gateway/JobGatewayBootApplication.java b/...ckend/job-gateway/src/main/java/com/tencent/bk/job/gateway/JobGatewayBootApplication.java
@@ -30,6 +30,7 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.web.embedded.NettyWebServerFactoryCustomizer;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.boot.web.embedded.netty.NettyReactiveWebServerFactory;
@@ -43,7 +44,8 @@
 /**
  * @date 2019/09/20
  */
-@SpringBootApplication(scanBasePackages = "com.tencent.bk.job")
+@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude =
+    {ApplicationAvailabilityAutoConfiguration.class})
 @EnableDiscoveryClient
 @Slf4j
 @EnableConfigurationProperties({FeatureToggleConfig.class})

diff --git a/...logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/JobLogBootApplication.java b/...logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/JobLogBootApplication.java
@@ -27,13 +27,15 @@
 import com.tencent.bk.job.common.config.FeatureToggleConfig;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 
 /**
  * @date 2019/09/20
  */
-@SpringBootApplication(scanBasePackages = "com.tencent.bk.job")
+@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude =
+    {ApplicationAvailabilityAutoConfiguration.class})
 @EnableFeignClients
 @EnableConfigurationProperties({FeatureToggleConfig.class})
 public class JobLogBootApplication {

diff --git a/...age/boot-job-manage/src/main/java/com/tencent/bk/job/manage/JobManageBootApplication.java b/...age/boot-job-manage/src/main/java/com/tencent/bk/job/manage/JobManageBootApplication.java
@@ -27,13 +27,15 @@
 import com.tencent.bk.job.common.config.FeatureToggleConfig;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration;
 import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.scheduling.annotation.EnableScheduling;
 
-@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class})
+@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class,
+    ApplicationAvailabilityAutoConfiguration.class})
 @EnableConfigurationProperties({FeatureToggleConfig.class})
 @EnableCaching
 @EnableFeignClients

diff --git a/support-files/kubernetes/images/backend/backend.Dockerfile b/support-files/kubernetes/images/backend/backend.Dockerfile
@@ -1,4 +1,4 @@
-FROM bkjob/jdk:0.0.2
+FROM bkjob/jdk:0.0.3
 
 LABEL maintainer="Tencent BlueKing Job"
 

diff --git a/support-files/kubernetes/images/backend/jdk.Dockerfile b/support-files/kubernetes/images/backend/jdk.Dockerfile
@@ -1,4 +1,4 @@
-FROM bkjob/os:0.0.1
+FROM bkjob/os:0.0.2
 
 LABEL maintainer="Tencent BlueKing Job"
 
@@ -10,4 +10,4 @@ RUN mkdir -p /data && \
     rm -f TencentKona8.0.3.b2_jdk_linux-x86_64_8u262.tar.gz
 ENV JAVA_HOME=/data/TencentKona-8.0.3-262
 ENV PATH=${JAVA_HOME}/bin:$PATH
-ENV CLASSPATH=.:${JAVA_HOME}/lib
+ENV CLASSPATH=.:${JAVA_HOME}/lib
diff --git a/support-files/kubernetes/images/backend/os.Dockerfile b/support-files/kubernetes/images/backend/os.Dockerfile
@@ -14,4 +14,6 @@ RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime &&\
 RUN yum install -y vim && \
     yum install -y less && \
     yum install -y wget && \
-    yum install -y lrzsz
+    yum install -y lrzsz && \
+    yum install -y net-tools.x86_64 && \
+    yum install -y bind-utils
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
		com.tencent.bk.job.common.consul.config.JobApplicationAvailabilityAutoConfiguration