Merge pull request TencentBlueKing#3197 from TencentBlueKing/master

merge: master to 3.10.x

src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImpl.java

@@ -232,14 +232,6 @@ private PageData<TaskInstanceDTO> listPageTaskInstanceByBasicInfo(TaskInstanceQu
         int start = baseSearchCondition.getStartOrDefault(0);
         int length = baseSearchCondition.getLengthOrDefault(10);
 
-        int count = 0;
-        if (baseSearchCondition.isCountPageTotal()) {
-            count = getPageTaskInstanceCount(taskQuery);
-            if (count == 0) {
-                return PageData.emptyPageData(start, length);
-            }
-        }
-
         Collection<SortField<?>> orderFields = new ArrayList<>();
         orderFields.add(TASK_INSTANCE.CREATE_TIME.desc());
         Result<?> result = ctx.select(TASK_INSTANCE.ID, TASK_INSTANCE.TASK_ID, TASK_INSTANCE.CRON_TASK_ID,
@@ -255,6 +247,12 @@ private PageData<TaskInstanceDTO> listPageTaskInstanceByBasicInfo(TaskInstanceQu
             .orderBy(orderFields)
             .limit(start, length)
             .fetch();
+
+        int count = 0;
+        if (baseSearchCondition.isCountPageTotal()) {
+            count = getPageTaskInstanceCount(taskQuery);
+        }
+
         return buildTaskInstancePageData(start, length, count, result);
     }
 
@@ -268,16 +266,7 @@ private PageData<TaskInstanceDTO> listPageTaskInstanceByIp(TaskInstanceQuery tas
         }
         int start = baseSearchCondition.getStartOrDefault(0);
         int length = baseSearchCondition.getLengthOrDefault(10);
-        int count = 0;
-        if (baseSearchCondition.isCountPageTotal()) {
-            count = ctx.selectCount().from(TaskInstance.TASK_INSTANCE)
-                .leftJoin(TASK_INSTANCE_HOST).on(TaskInstance.TASK_INSTANCE.ID.eq(TASK_INSTANCE_HOST.TASK_INSTANCE_ID))
-                .where(conditions)
-                .fetchOne(0, Integer.class);
-            if (count == 0) {
-                return PageData.emptyPageData(start, length);
-            }
-        }
+
         Collection<SortField<?>> orderFields = new ArrayList<>();
         orderFields.add(TASK_INSTANCE.ID.desc());
         Result result = ctx.select(TASK_INSTANCE.ID, TASK_INSTANCE.TASK_ID, TASK_INSTANCE.CRON_TASK_ID,
@@ -295,6 +284,17 @@ private PageData<TaskInstanceDTO> listPageTaskInstanceByIp(TaskInstanceQuery tas
             .orderBy(orderFields)
             .limit(start, length)
             .fetch();
+
+        int count = 0;
+        if (baseSearchCondition.isCountPageTotal()) {
+            count = ctx.selectCount()
+                .from(TaskInstance.TASK_INSTANCE)
+                .leftJoin(TASK_INSTANCE_HOST)
+                .on(TaskInstance.TASK_INSTANCE.ID.eq(TASK_INSTANCE_HOST.TASK_INSTANCE_ID))
+                .where(conditions)
+                .fetchOne(0, Integer.class);
+        }
+
         return buildTaskInstancePageData(start, length, count, result);
     }
 

src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/DangerousRuleServiceImpl.java

@@ -103,11 +103,10 @@ public DangerousRuleDTO updateDangerousRule(String username, AddOrUpdateDangerou
             dangerousRuleDAO.updateDangerousRule(new DangerousRuleDTO(req.getId(),
                 req.getExpression(), req.getDescription(), existDangerousRuleDTO.getPriority(), scriptType, null,
                 null, username, System.currentTimeMillis(), req.getAction(), req.getStatus()));
+            // 清理缓存
+            List<Byte> existScriptTypes  = DangerousRuleDTO.decodeScriptType(existDangerousRuleDTO.getScriptType());
+            dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes(existScriptTypes);
         }
-
-        // 清理缓存
-        dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes(req.getScriptTypeList());
-
         return getDangerousRuleById(req.getId());
     }
 

docs/apidoc/bk-api-gateway/v3/bin/sync-bkapigateway.sh → support-files/bk-api-gateway/v3/bin/sync-bkapigateway.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 # 加载 apigw-manager 原始镜像中的通用函数
 source /apigw-manager/bin/functions.sh
@@ -7,12 +8,13 @@ source /apigw-manager/bin/functions.sh
 # - 如在下面指令的参数中，指定了参数 --gateway-name=${gateway_name}，则使用该参数指定的网关名
 # - 如在下面指令的参数中，未指定参数 --gateway-name，则使用 Django settings BK_APIGW_NAME
 gateway_name=$BK_APIGW_NAME
+# 自动发布资源，create_version_and_release_apigw命令中指定参数 --no-pub只生成版本不发布资源，不指定生成版本且发布资源
+gateway_auto_publish=$BK_APIGW_AUTO_PUBLISH
 
 title "do something before migrate"
 # 网关维护人员解析成array格式
 BK_APIGW_MAINTAINERS_ARRAY='["'${BK_APIGW_MAINTAINERS//,/\",\"}'"]'
 export BK_APIGW_MAINTAINERS="$BK_APIGW_MAINTAINERS_ARRAY"
-echo "maintainers $BK_APIGW_MAINTAINERS"
 
 # 待同步网关、资源定义文件
 definition_file="/data/definition.yaml"
@@ -32,6 +34,12 @@ title "fetch apigateway public key"
 apigw-manager.sh fetch_apigw_public_key --gateway-name=${gateway_name} --print > "apigateway.pub"
 
 title "releasing"
-call_definition_command_or_exit create_version_and_release_apigw "${definition_file}" --gateway-name=${gateway_name}
+if [ "$gateway_auto_publish" = "true" ]; then
+  echo "Generate versions and automatically publish resources to blueking api gateway"
+  call_definition_command_or_exit create_version_and_release_apigw "${definition_file}" --gateway-name=${gateway_name}
+else
+  echo "Only generate version, manually publish after confirmation on the blueking api gateway management page"
+  call_definition_command_or_exit create_version_and_release_apigw "${definition_file}" --gateway-name=${gateway_name} --no-pub
+fi
 
 title "done"

docs/apidoc/bk-api-gateway/v3/definition.yaml → support-files/bk-api-gateway/v3/definition.yaml

@@ -7,9 +7,9 @@ release:
   # 发布版本号
   # 每次同步资源到网关会对比，如果资源有变更就会新增版本号，如果这里指定了版本号就用这里指定的，
   # 如果没有，就会用时间戳+上次的版本号生成一个，即上一次版本号+时间戳
-  version: 1.0.0
-  title: "1.0.0"
-  comment: "1.0.0"
+  version: 3.10.0
+  title: "3.10.0"
+  comment: "3.10.0"
 
 # 定义网关基本信息，用于命令 `sync_apigw_config`
 apigateway:

support-files/kubernetes/charts/bk-job/VALUES_LOG.md

@@ -1,5 +1,33 @@
 # chart values 更新日志
 
+## 0.7.0
+1. 增加接入蓝鲸网关配置
+
+```yaml
+bkApiGatewayConfig:
+  # 是否自动把API注册到蓝鲸网关
+  sync: false
+  # 是否自动发布资源，true：生成版本且发布资源，false：只生成版本不发布资源
+  autoPublish: true
+  # 是否开启apigw jwt认证
+  enabled: true
+  jwtPublicKey:
+    # jwtPublicKey获取策略，获取失败重试：retry, 获取失败终止启动：abort
+    failPolicy: "retry"
+  # 接入的网关名称，蓝鲸官方网关都是bk-开头
+  gatewayName: "bk-job"
+  # 同步蓝鲸网关url
+  syncUrl: "http://bkapi.example.com/api/{api_name}"
+  # 蓝鲸网关接口url
+  apiUrl: "http://bkapi.example.com/api/{api_name}/{env}"
+  # 接入环境
+  stage: "prod"
+  # 接入的api资源目录, 默认是/data/apidocs/
+  resourceDir: "/data/apidocs/"
+  # 网关维护人员，仅维护人员有管理网关的权限, 多个维护人员逗号分隔，如:"user1,user2"
+  maintainers: "admin"
+```
+
 ## 0.6.5
 1. 增加正在执行中的作业总量的配额限制
 
@@ -64,6 +92,19 @@ job:
 bkCmdbApiGatewayUrl: "http://bkapi.example.com/api/cmdb"
 ```
 
+## 0.5.10
+1. 增加MySQL migration的自定义数据库账号和密码
+
+```yaml
+job:
+  migration:
+    mysqlSchema:
+      # mysql数据库migrate时使用的管理员用户名，不填默认是root
+      adminUsername: ""
+      # mysql数据库migrate时使用的管理员密码，不填使用mariadb/externalMariaDB的root密码
+      adminPassword: ""
+```
+
 ## 0.5.9
 
 1. 新增备份服务中的数据归档相关配置

support-files/kubernetes/charts/bk-job/templates/_helpers.tpl

@@ -214,6 +214,26 @@ Return the MariaDB root password
 {{- end -}}
 {{- end -}}
 
+{{/*
+Return the migrate mysqlSchema admin username
+*/}}
+{{- define "job.migration.mysqlSchema.adminUsername" -}}
+{{- printf "%s" .Values.job.migration.mysqlSchema.adminUsername | default "root" -}}
+{{- end -}}
+
+{{/*
+Return the migrate mysqlSchema admin password
+*/}}
+{{- define "job.migration.mysqlSchema.adminPassword" -}}
+{{- if .Values.job.migration.mysqlSchema.adminPassword -}}
+    {{- printf "%s" .Values.job.migration.mysqlSchema.adminPassword -}}
+{{- else if .Values.externalMariaDB.rootPassword -}}
+    {{- printf "%s" .Values.externalMariaDB.rootPassword -}}
+{{- else -}}
+    {{- printf "%s" .Values.mariadb.auth.rootPassword -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Return the MariaDB secret name
 */}}

support-files/kubernetes/charts/bk-job/templates/job-migration/job.yaml

@@ -45,8 +45,16 @@ spec:
               value: {{ include "job.mariadb.host" . | quote }}
             - name: BK_JOB_MYSQL_PORT
               value: {{ include "job.mariadb.port" . | quote }}
-            - name: BK_JOB_MYSQL_ROOT_PASSWORD
-              value: {{ include "job.mariadb.rootPassword" . | quote }}
+            - name: BK_JOB_MYSQL_ADMIN_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{printf "%s-%s" (include "job.fullname" .) "migration-db"}}
+                  key: admin-username
+            - name: BK_JOB_MYSQL_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ printf "%s-%s" (include "job.fullname" .) "migration-db" }}
+                  key: admin-password
             - name: BK_IAM_URL
               value: {{ .Values.bkIamApiUrl | quote }}
             - name: BK_JOB_APP_CODE

support-files/kubernetes/charts/bk-job/templates/job-migration/sync-bkapigateway-job.yaml

@@ -43,5 +43,7 @@ spec:
           value: "{{ .Values.bkDomainScheme }}://{{ .Values.job.web.domain }}"
         - name: BK_APIGW_MAINTAINERS
           value: {{ .Values.bkApiGatewayConfig.maintainers }}
+        - name: BK_APIGW_AUTO_PUBLISH
+          value: {{.Values.bkApiGatewayConfig.autoPublish | quote }}
       restartPolicy: Never
 {{- end }}

support-files/kubernetes/charts/bk-job/templates/secret-migrationDB.yaml

@@ -0,0 +1,10 @@
+{{- if .Values.migration.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ printf "%s-%s" (include "job.fullname" .) "migration-db" }}
+data:
+  admin-username: {{ include "job.migration.mysqlSchema.adminUsername" . | b64enc | quote }}
+  admin-password: {{ include "job.migration.mysqlSchema.adminPassword" . | b64enc | quote }}
+type: Opaque
+{{- end }}

support-files/kubernetes/charts/bk-job/values.yaml

@@ -645,6 +645,10 @@ job:
     mysqlSchema:
       # 是否开启Mysql数据库结构migration
       enabled: true
+      # mysql数据库migrate时使用的管理员用户名，不填默认是root
+      adminUsername: ""
+      # mysql数据库migrate时使用的管理员密码，不填使用mariadb/externalMariaDB的root密码
+      adminPassword: ""
   # 各微服务Actuator监控端口
   managePort: 19876
   web:
@@ -1512,6 +1516,8 @@ migrationInit:
 bkApiGatewayConfig:
   # 是否自动把API注册到蓝鲸网关
   sync: false
+  # 是否自动发布资源，true：生成版本且发布资源，false：只生成版本不发布资源
+  autoPublish: true
   # 是否开启apigw jwt认证
   enabled: true
   jwtPublicKey:

support-files/kubernetes/images/build.sh

@@ -259,7 +259,7 @@ build_migration_image(){
 build_sync_bk_api_gateway_image(){
     log "Building sync_bk_api_gateway image, version: ${VERSION}..."
     rm -rf tmp/sync_bk_api_gateway/*
-    cp -r $ROOT_DIR/docs/apidoc/bk-api-gateway/v3/* tmp/sync_bk_api_gateway/
+    cp -r $ROOT_DIR/support-files/bk-api-gateway/v3/* tmp/sync_bk_api_gateway/
     docker build -f migration/bkApiGateway.Dockerfile -t $REGISTRY/job-sync-bk-api-gateway:$VERSION tmp/sync_bk_api_gateway --network=host
     if [[ $PUSH -eq 1 ]] ; then
         docker push $REGISTRY/job-sync-bk-api-gateway:$VERSION

support-files/kubernetes/images/migration/bkApiGateway.Dockerfile

@@ -1,4 +1,4 @@
-FROM hub.bktencent.com/blueking/apigw-manager:3.0.4
+FROM hub.bktencent.com/blueking/apigw-manager:3.1.1
 
 COPY bin /data/bin
 COPY apidocs /data/apidocs

support-files/kubernetes/images/migration/startup.sh

@@ -10,7 +10,7 @@ echo "===========EXEC========"
 mysql --version
 
 function checkMysql(){
-  c=$(mysql -h $BK_JOB_MYSQL_HOST -P $BK_JOB_MYSQL_PORT -uroot -p$BK_JOB_MYSQL_ROOT_PASSWORD -e "select 1"|grep 1|wc -l)
+  c=$(mysql -h $BK_JOB_MYSQL_HOST -P $BK_JOB_MYSQL_PORT -u$BK_JOB_MYSQL_ADMIN_USERNAME -p$BK_JOB_MYSQL_ADMIN_PASSWORD -e "select 1"|grep 1|wc -l)
   echo "c=$c"
   if [[ "$c" == "2" ]];then
     return 0
@@ -30,7 +30,7 @@ function migrateMySQL(){
   done
 
   for sql in "${ALL_SQL[@]}"; do
-    mysql -h $BK_JOB_MYSQL_HOST -P $BK_JOB_MYSQL_PORT -uroot -p$BK_JOB_MYSQL_ROOT_PASSWORD < $sql
+    mysql -h $BK_JOB_MYSQL_HOST -P $BK_JOB_MYSQL_PORT -u$BK_JOB_MYSQL_ADMIN_USERNAME -p$BK_JOB_MYSQL_ADMIN_PASSWORD < $sql
   done
 }