From 8b4a92820d1c4b95b05bfaeca82117acd30c4d45 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Mon, 6 Dec 2021 15:20:56 +0800 Subject: [PATCH 01/35] =?UTF-8?q?perf:=20=E5=AE=B9=E5=99=A8=E5=8C=96?= =?UTF-8?q?=E5=90=84=E5=BE=AE=E6=9C=8D=E5=8A=A1=E6=97=A5=E5=BF=97=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E9=87=87=E7=94=A8PVC=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=8F=E8=BF=87=E5=A4=A7=E5=AF=BC=E8=87=B4?= =?UTF-8?q?Pod=E8=A2=AB=E9=A9=B1=E9=80=90=20#507?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加PV-PVC，日志目录与执行目录分离 --- support-files/kubernetes/charts/bk-job/README.md | 2 +- .../bk-job/templates/job-analysis/deployment.yaml | 9 +++++++-- .../charts/bk-job/templates/pv-default.yaml | 14 ++++++++++++++ .../charts/bk-job/templates/pvc-storage.yaml | 11 +++++++++++ support-files/kubernetes/charts/bk-job/values.yaml | 2 +- .../kubernetes/images/backend/backend.Dockerfile | 12 ++++++------ support-files/kubernetes/images/backend/startup.sh | 2 +- .../images/file-worker/fileWorker.Dockerfile | 10 +++++----- .../kubernetes/images/file-worker/startup.sh | 2 +- .../images/migration/migration.Dockerfile | 12 ++++++------ 10 files changed, 53 insertions(+), 23 deletions(-) create mode 100644 support-files/kubernetes/charts/bk-job/templates/pv-default.yaml create mode 100644 support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml diff --git a/support-files/kubernetes/charts/bk-job/README.md b/support-files/kubernetes/charts/bk-job/README.md index 3d6d85cc5a..57bc4acead 100644 --- a/support-files/kubernetes/charts/bk-job/README.md +++ b/support-files/kubernetes/charts/bk-job/README.md @@ -88,7 +88,7 @@ bitnami/rabbitmq | `job.security.actuator.user.name` | actuator管理账号 | `actuator_name` | | `job.security.actuator.user.password` | actuator管理密码 | `actuator_password` | | `job.encrypt.password` | 加密DB密码/凭证的对称密钥 | `encrypt_password` | -| `job.storage.rootPath` | 本地文件下载暂存路径 | `/data/job/local` | +| `job.storage.rootPath` | 本地文件下载暂存路径 | `/data/job/storage/local` | ### 微服务网关Job-Gateway配置 |参数|描述|默认值 | diff --git a/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml index 0b38c78311..fe0c942dad 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml @@ -70,9 +70,9 @@ spec: - name: BK_JOB_JAR value: job-analysis.jar - name: BK_JOB_LOG_BASE_DIR - value: /data/logs/job + value: /data/job/storage/logs - name: BK_JOB_LOG_DIR - value: /data/logs/job/job-analysis + value: /data/job/storage/logs/job-analysis - name: JOB_APPLICATION_CONFIGMAP_NAME value: {{ include "common.names.fullname" . }}-analysis - name: JOB_COMMON_CONFIGMAP_NAME @@ -108,6 +108,8 @@ spec: resources: {{- toYaml .Values.analysisConfig.resources | nindent 12 }} volumeMounts: + - name: job-storage + mountPath: /data/job/storage - name: mariadb mountPath: /etc/secrets/mariadb readOnly: true @@ -118,6 +120,9 @@ spec: mountPath: /etc/secrets/redis readOnly: true volumes: + - name: job-storage + persistentVolumeClaim: + claimName: bk-job-pv-claim - name: mariadb secret: secretName: {{ include "job.mariadb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml b/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml new file mode 100644 index 0000000000..abb4012fc6 --- /dev/null +++ b/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: bk-job-local-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 200Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/data/bkjob" diff --git a/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml b/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml new file mode 100644 index 0000000000..5f101ba655 --- /dev/null +++ b/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: bk-job-pv-claim +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 200Gi diff --git a/support-files/kubernetes/charts/bk-job/values.yaml b/support-files/kubernetes/charts/bk-job/values.yaml index 452984105e..e7f46c24c0 100644 --- a/support-files/kubernetes/charts/bk-job/values.yaml +++ b/support-files/kubernetes/charts/bk-job/values.yaml @@ -472,7 +472,7 @@ job: fileManage: enabled: true storage: - rootPath: /data/job/local + rootPath: /data/job/storage/local # 蓝鲸日志平台数据采集 log: file: diff --git a/support-files/kubernetes/images/backend/backend.Dockerfile b/support-files/kubernetes/images/backend/backend.Dockerfile index 5af352a2ed..8add28f648 100644 --- a/support-files/kubernetes/images/backend/backend.Dockerfile +++ b/support-files/kubernetes/images/backend/backend.Dockerfile @@ -2,17 +2,17 @@ FROM blueking/jdk:0.0.1 LABEL maintainer="Tencent BlueKing Job" -ENV BK_JOB_HOME=/data/job \ - BK_JOB_LOGS_DIR=/data/job/logs +ENV BK_JOB_HOME=/data/job/exec \ + BK_JOB_LOGS_DIR=/data/job/storage/logs -COPY ./ /data/job/ +COPY ./ /data/job/exec/ RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ echo 'Asia/Shanghai' > /etc/timezone && \ - chmod +x /data/job/startup.sh + chmod +x /data/job/exec/startup.sh ENV LANG en_US.utf8 ENV LANGUAGE en_US.utf8 ENV LC_ALL en_US.utf8 -WORKDIR /data/job -CMD /data/job/startup.sh +WORKDIR /data/job/exec +CMD /data/job/exec/startup.sh diff --git a/support-files/kubernetes/images/backend/startup.sh b/support-files/kubernetes/images/backend/startup.sh index 9ae1f94b52..922e8c964e 100644 --- a/support-files/kubernetes/images/backend/startup.sh +++ b/support-files/kubernetes/images/backend/startup.sh @@ -25,4 +25,4 @@ java -server \ -XX:ErrorFile=$BK_JOB_LOG_DIR/error_sys.log \ -Dspring.profiles.active=$BK_JOB_PROFILE \ $BK_JOB_JVM_OPTION \ - -jar /data/job/$BK_JOB_JAR + -jar /data/job/exec/$BK_JOB_JAR diff --git a/support-files/kubernetes/images/file-worker/fileWorker.Dockerfile b/support-files/kubernetes/images/file-worker/fileWorker.Dockerfile index 5af352a2ed..e4b9eb4360 100644 --- a/support-files/kubernetes/images/file-worker/fileWorker.Dockerfile +++ b/support-files/kubernetes/images/file-worker/fileWorker.Dockerfile @@ -2,17 +2,17 @@ FROM blueking/jdk:0.0.1 LABEL maintainer="Tencent BlueKing Job" -ENV BK_JOB_HOME=/data/job \ +ENV BK_JOB_HOME=/data/job/exec \ BK_JOB_LOGS_DIR=/data/job/logs -COPY ./ /data/job/ +COPY ./ /data/job/exec/ RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ echo 'Asia/Shanghai' > /etc/timezone && \ - chmod +x /data/job/startup.sh + chmod +x /data/job/exec/startup.sh ENV LANG en_US.utf8 ENV LANGUAGE en_US.utf8 ENV LC_ALL en_US.utf8 -WORKDIR /data/job -CMD /data/job/startup.sh +WORKDIR /data/job/exec +CMD /data/job/exec/startup.sh diff --git a/support-files/kubernetes/images/file-worker/startup.sh b/support-files/kubernetes/images/file-worker/startup.sh index 4898ff0fb6..8ff4d2870b 100644 --- a/support-files/kubernetes/images/file-worker/startup.sh +++ b/support-files/kubernetes/images/file-worker/startup.sh @@ -31,4 +31,4 @@ java -server \ -XX:ErrorFile=$BK_JOB_LOG_DIR/error_sys.log \ -Dspring.profiles.active=$BK_JOB_PROFILE \ $BK_JOB_JVM_OPTION \ - -jar /data/job/$BK_JOB_JAR + -jar /data/job/exec/$BK_JOB_JAR diff --git a/support-files/kubernetes/images/migration/migration.Dockerfile b/support-files/kubernetes/images/migration/migration.Dockerfile index 2580df9384..b89304848a 100644 --- a/support-files/kubernetes/images/migration/migration.Dockerfile +++ b/support-files/kubernetes/images/migration/migration.Dockerfile @@ -2,10 +2,10 @@ FROM blueking/jdk:0.0.1 LABEL maintainer="Tencent BlueKing Job" -ENV BK_JOB_HOME=/data/job \ - BK_JOB_LOGS_DIR=/data/job/logs +ENV BK_JOB_HOME=/data/job/exec \ + BK_JOB_LOGS_DIR=/data/job/storage/logs -COPY ./ /data/job/ +COPY ./ /data/job/exec/ RUN yum -y install mysql RUN yum install -y epel-release @@ -14,11 +14,11 @@ RUN pip install requests RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ echo 'Asia/Shanghai' > /etc/timezone && \ - chmod +x /data/job/startup.sh + chmod +x /data/job/exec/startup.sh ENV LANG en_US.utf8 ENV LANGUAGE en_US.utf8 ENV LC_ALL en_US.utf8 -WORKDIR /data/job -CMD /data/job/startup.sh +WORKDIR /data/job/exec +CMD /data/job/exec/startup.sh From 920c640b78ff3a3899b4fa02d3d6d9b49388837c Mon Sep 17 00:00:00 2001 From: jsonwan Date: Mon, 6 Dec 2021 16:29:27 +0800 Subject: [PATCH 02/35] =?UTF-8?q?perf:=20=E5=AE=B9=E5=99=A8=E5=8C=96?= =?UTF-8?q?=E5=90=84=E5=BE=AE=E6=9C=8D=E5=8A=A1=E6=97=A5=E5=BF=97=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E9=87=87=E7=94=A8PVC=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=8F=E8=BF=87=E5=A4=A7=E5=AF=BC=E8=87=B4?= =?UTF-8?q?Pod=E8=A2=AB=E9=A9=B1=E9=80=90=20#507?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pod日志路径隔离 --- .../charts/bk-job/templates/job-analysis/deployment.yaml | 8 ++++++++ support-files/kubernetes/images/backend/startup.sh | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml index fe0c942dad..787e7960a5 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml @@ -63,12 +63,16 @@ spec: image: {{ include "job-analysis.image" . }} imagePullPolicy: {{ .Values.analysisConfig.image.pullPolicy }} env: + - name: BK_JOB_APP_NAME + value: "job-analysis" - name: BK_JOB_JVM_OPTION value: {{ .Values.analysisConfig.jvmOptions | quote }} - name: BK_JOB_PROFILE value: {{ include "job.profile" . }} - name: BK_JOB_JAR value: job-analysis.jar + - name: BK_JOB_STORAGE_BASE_DIR + value: /data/job/storage - name: BK_JOB_LOG_BASE_DIR value: /data/job/storage/logs - name: BK_JOB_LOG_DIR @@ -83,6 +87,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: BK_JOB_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.analysisConfig.containerPort }} diff --git a/support-files/kubernetes/images/backend/startup.sh b/support-files/kubernetes/images/backend/startup.sh index 922e8c964e..2593982d48 100644 --- a/support-files/kubernetes/images/backend/startup.sh +++ b/support-files/kubernetes/images/backend/startup.sh @@ -1,5 +1,14 @@ #! /bin/sh +echo "BK_JOB_APP_NAME=$BK_JOB_APP_NAME" +echo "KUBERNETES_NAMESPACE=$KUBERNETES_NAMESPACE" +echo "BK_JOB_POD_NAME=$BK_JOB_POD_NAME" +echo "BK_JOB_STORAGE_BASE_DIR=$BK_JOB_STORAGE_BASE_DIR" + +# 拼接日志相关路径 +BK_JOB_LOG_BASE_DIR="$BK_JOB_STORAGE_BASE_DIR/$KUBERNETES_NAMESPACE/$BK_JOB_POD_NAME/logs" +BK_JOB_LOG_DIR="$BK_JOB_LOG_BASE_DIR/$BK_JOB_APP_NAME" + echo "BK_JOB_LOG_BASE_DIR=$BK_JOB_LOG_BASE_DIR" mkdir -p "$BK_JOB_LOG_BASE_DIR" chmod 777 "$BK_JOB_LOG_BASE_DIR" From 4fb547562b3c1fd18fe035b2458ac06fbc56f84a Mon Sep 17 00:00:00 2001 From: jsonwan Date: Mon, 6 Dec 2021 17:10:07 +0800 Subject: [PATCH 03/35] =?UTF-8?q?perf:=20=E5=AE=B9=E5=99=A8=E5=8C=96?= =?UTF-8?q?=E5=90=84=E5=BE=AE=E6=9C=8D=E5=8A=A1=E6=97=A5=E5=BF=97=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E9=87=87=E7=94=A8PVC=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=8F=E8=BF=87=E5=A4=A7=E5=AF=BC=E8=87=B4?= =?UTF-8?q?Pod=E8=A2=AB=E9=A9=B1=E9=80=90=20#507?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 各微服务日志路径更新 --- support-files/kubernetes/charts/bk-job/README.md | 2 +- .../bk-job/templates/job-backup/deployment.yaml | 11 +++++++++++ .../bk-job/templates/job-crontab/deployment.yaml | 11 +++++++++++ .../bk-job/templates/job-execute/deployment.yaml | 11 +++++++++++ .../bk-job/templates/job-file-gateway/deployment.yaml | 11 +++++++++++ .../bk-job/templates/job-file-worker/statefulset.yaml | 11 +++++++++++ .../bk-job/templates/job-gateway/deployment.yaml | 11 +++++++++++ .../bk-job/templates/job-logsvr/deployment.yaml | 11 +++++++++++ .../bk-job/templates/job-manage/deployment.yaml | 11 +++++++++++ .../charts/bk-job/templates/pv-default.yaml | 2 +- .../charts/bk-job/templates/pvc-storage.yaml | 2 +- support-files/kubernetes/charts/bk-job/values.yaml | 2 +- .../kubernetes/images/file-worker/startup.sh | 9 +++++++++ 13 files changed, 101 insertions(+), 4 deletions(-) diff --git a/support-files/kubernetes/charts/bk-job/README.md b/support-files/kubernetes/charts/bk-job/README.md index 57bc4acead..3d6d85cc5a 100644 --- a/support-files/kubernetes/charts/bk-job/README.md +++ b/support-files/kubernetes/charts/bk-job/README.md @@ -88,7 +88,7 @@ bitnami/rabbitmq | `job.security.actuator.user.name` | actuator管理账号 | `actuator_name` | | `job.security.actuator.user.password` | actuator管理密码 | `actuator_password` | | `job.encrypt.password` | 加密DB密码/凭证的对称密钥 | `encrypt_password` | -| `job.storage.rootPath` | 本地文件下载暂存路径 | `/data/job/storage/local` | +| `job.storage.rootPath` | 本地文件下载暂存路径 | `/data/job/local` | ### 微服务网关Job-Gateway配置 |参数|描述|默认值 | diff --git a/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml index b89fb14a2e..4f01c2a6c6 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml @@ -63,6 +63,8 @@ spec: image: {{ include "job-backup.image" . }} imagePullPolicy: {{ .Values.backupConfig.image.pullPolicy }} env: + - name: BK_JOB_APP_NAME + value: "job-backup" - name: BK_JOB_JVM_OPTION value: {{ .Values.backupConfig.jvmOptions | quote }} - name: BK_JOB_PROFILE @@ -83,6 +85,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: BK_JOB_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.backupConfig.containerPort }} @@ -108,6 +114,8 @@ spec: resources: {{- toYaml .Values.backupConfig.resources | nindent 12 }} volumeMounts: + - name: job-storage + mountPath: /data/job/storage - name: mariadb mountPath: /etc/secrets/mariadb readOnly: true @@ -118,6 +126,9 @@ spec: mountPath: /etc/secrets/redis readOnly: true volumes: + - name: job-storage + persistentVolumeClaim: + claimName: bk-job-pv-claim - name: mariadb secret: secretName: {{ include "job.mariadb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml index 3704a5f6cc..f493a53fb2 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml @@ -64,6 +64,8 @@ spec: image: {{ include "job-crontab.image" . }} imagePullPolicy: {{ .Values.crontabConfig.image.pullPolicy }} env: + - name: BK_JOB_APP_NAME + value: "job-crontab" - name: BK_JOB_JVM_OPTION value: {{ .Values.crontabConfig.jvmOptions | quote }} - name: BK_JOB_PROFILE @@ -84,6 +86,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: BK_JOB_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.crontabConfig.containerPort }} @@ -109,6 +115,8 @@ spec: resources: {{- toYaml .Values.crontabConfig.resources | nindent 12 }} volumeMounts: + - name: job-storage + mountPath: /data/job/storage - name: mariadb mountPath: /etc/secrets/mariadb readOnly: true @@ -119,6 +127,9 @@ spec: mountPath: /etc/secrets/redis readOnly: true volumes: + - name: job-storage + persistentVolumeClaim: + claimName: bk-job-pv-claim - name: mariadb secret: secretName: {{ include "job.mariadb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml index 9b36a2da42..9cea519e06 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml @@ -63,6 +63,8 @@ spec: image: {{ include "job-execute.image" . }} imagePullPolicy: {{ .Values.executeConfig.image.pullPolicy }} env: + - name: BK_JOB_APP_NAME + value: "job-execute" - name: BK_JOB_JVM_OPTION value: {{ .Values.executeConfig.jvmOptions | quote }} - name: BK_JOB_PROFILE @@ -89,6 +91,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: BK_JOB_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.executeConfig.containerPort }} @@ -114,6 +120,8 @@ spec: resources: {{- toYaml .Values.executeConfig.resources | nindent 12 }} volumeMounts: + - name: job-storage + mountPath: /data/job/storage - name: local-file-dir mountPath: {{ .Values.job.storage.rootPath }} - name: mariadb @@ -132,6 +140,9 @@ spec: mountPath: /etc/secrets readOnly: true volumes: + - name: job-storage + persistentVolumeClaim: + claimName: bk-job-pv-claim - name: local-file-dir hostPath: path: {{ .Values.job.storage.rootPath }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml index f1abfd8a6f..1feedcdc3c 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml @@ -63,6 +63,8 @@ spec: image: {{ include "job-file-gateway.image" . }} imagePullPolicy: {{ .Values.fileGatewayConfig.image.pullPolicy }} env: + - name: BK_JOB_APP_NAME + value: "job-file-gateway" - name: BK_JOB_JVM_OPTION value: {{ .Values.fileGatewayConfig.jvmOptions | quote }} - name: BK_JOB_PROFILE @@ -83,6 +85,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: BK_JOB_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.fileGatewayConfig.containerPort }} @@ -108,6 +114,8 @@ spec: resources: {{- toYaml .Values.fileGatewayConfig.resources | nindent 12 }} volumeMounts: + - name: job-storage + mountPath: /data/job/storage - name: mariadb mountPath: /etc/secrets/mariadb readOnly: true @@ -118,6 +126,9 @@ spec: mountPath: /etc/secrets/redis readOnly: true volumes: + - name: job-storage + persistentVolumeClaim: + claimName: bk-job-pv-claim - name: mariadb secret: secretName: {{ include "job.mariadb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml index f284962d4b..b18a969f8b 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml @@ -62,6 +62,8 @@ spec: image: {{ include "job-file-worker.image" . }} imagePullPolicy: {{ .Values.fileWorkerConfig.image.pullPolicy }} env: + - name: BK_JOB_APP_NAME + value: "job-file-worker" - name: BK_JOB_JVM_OPTION value: {{ .Values.fileWorkerConfig.jvmOptions | quote }} - name: BK_JOB_PROFILE @@ -94,6 +96,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: BK_JOB_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.fileWorkerConfig.containerPort }} @@ -119,9 +125,14 @@ spec: resources: {{- toYaml .Values.fileWorkerConfig.resources | nindent 12 }} volumeMounts: + - name: job-storage + mountPath: /data/job/storage - name: workspace mountPath: {{ .Values.fileWorkerConfig.downloadFile.dir }}/JobFileWorkerWorkspace volumes: + - name: job-storage + persistentVolumeClaim: + claimName: bk-job-pv-claim - name: workspace hostPath: path: {{ .Values.fileWorkerConfig.downloadFile.dir }}/JobFileWorkerWorkspace diff --git a/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml index 2517face0b..3f6519c9a6 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml @@ -61,6 +61,8 @@ spec: image: {{ include "job-gateway.image" . }} imagePullPolicy: {{ .Values.gatewayConfig.image.pullPolicy }} env: + - name: BK_JOB_APP_NAME + value: "job-gateway" - name: BK_JOB_JVM_OPTION value: {{ .Values.gatewayConfig.jvmOptions | quote }} - name: BK_JOB_PROFILE @@ -81,6 +83,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: BK_JOB_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.gatewayConfig.containerPort.http }} @@ -112,6 +118,8 @@ spec: resources: {{- toYaml .Values.gatewayConfig.resources | nindent 12 }} volumeMounts: + - name: job-storage + mountPath: /data/job/storage - name: job-gateway-tls-certs mountPath: /data/job/cert readOnly: true @@ -119,6 +127,9 @@ spec: mountPath: /etc/secrets readOnly: true volumes: + - name: job-storage + persistentVolumeClaim: + claimName: bk-job-pv-claim - name: job-gateway-tls-certs secret: secretName: {{ printf "%s-gateway-%s" (include "common.names.fullname" .) "tls-cert" }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml index dd181edb78..dd1834b2c6 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml @@ -61,6 +61,8 @@ spec: image: {{ include "job-logsvr.image" . }} imagePullPolicy: {{ .Values.logsvrConfig.image.pullPolicy }} env: + - name: BK_JOB_APP_NAME + value: "job-logsvr" - name: BK_JOB_JVM_OPTION value: {{ .Values.logsvrConfig.jvmOptions | quote }} - name: BK_JOB_PROFILE @@ -81,6 +83,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: BK_JOB_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.logsvrConfig.containerPort }} @@ -106,10 +112,15 @@ spec: resources: {{- toYaml .Values.logsvrConfig.resources | nindent 12 }} volumeMounts: + - name: job-storage + mountPath: /data/job/storage - name: mongodb mountPath: /etc/secrets/mongodb readOnly: true volumes: + - name: job-storage + persistentVolumeClaim: + claimName: bk-job-pv-claim - name: mongodb secret: secretName: {{ include "job.mongodb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml index 52ff545c85..5b86fe97fd 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml @@ -63,6 +63,8 @@ spec: image: {{ include "job-manage.image" . }} imagePullPolicy: {{ .Values.manageConfig.image.pullPolicy }} env: + - name: BK_JOB_APP_NAME + value: "job-manage" - name: BK_JOB_EDITION value: {{ .Values.job.edition }} - name: BK_JOB_JVM_OPTION @@ -85,6 +87,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: BK_JOB_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.manageConfig.containerPort }} @@ -110,6 +116,8 @@ spec: resources: {{- toYaml .Values.manageConfig.resources | nindent 12 }} volumeMounts: + - name: job-storage + mountPath: /data/job/storage - name: mariadb mountPath: /etc/secrets/mariadb readOnly: true @@ -126,6 +134,9 @@ spec: mountPath: /etc/secrets readOnly: true volumes: + - name: job-storage + persistentVolumeClaim: + claimName: bk-job-pv-claim - name: mariadb secret: secretName: {{ include "job.mariadb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml b/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml index abb4012fc6..b09df791ab 100644 --- a/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml @@ -9,6 +9,6 @@ spec: capacity: storage: 200Gi accessModes: - - ReadWriteOnce + - ReadWriteMany hostPath: path: "/data/bkjob" diff --git a/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml b/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml index 5f101ba655..74d90c5234 100644 --- a/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml @@ -5,7 +5,7 @@ metadata: spec: storageClassName: manual accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 200Gi diff --git a/support-files/kubernetes/charts/bk-job/values.yaml b/support-files/kubernetes/charts/bk-job/values.yaml index e7f46c24c0..452984105e 100644 --- a/support-files/kubernetes/charts/bk-job/values.yaml +++ b/support-files/kubernetes/charts/bk-job/values.yaml @@ -472,7 +472,7 @@ job: fileManage: enabled: true storage: - rootPath: /data/job/storage/local + rootPath: /data/job/local # 蓝鲸日志平台数据采集 log: file: diff --git a/support-files/kubernetes/images/file-worker/startup.sh b/support-files/kubernetes/images/file-worker/startup.sh index 8ff4d2870b..573ed2bf9a 100644 --- a/support-files/kubernetes/images/file-worker/startup.sh +++ b/support-files/kubernetes/images/file-worker/startup.sh @@ -1,5 +1,14 @@ #! /bin/sh +echo "BK_JOB_APP_NAME=$BK_JOB_APP_NAME" +echo "KUBERNETES_NAMESPACE=$KUBERNETES_NAMESPACE" +echo "BK_JOB_POD_NAME=$BK_JOB_POD_NAME" +echo "BK_JOB_STORAGE_BASE_DIR=$BK_JOB_STORAGE_BASE_DIR" + +# 拼接日志相关路径 +BK_JOB_LOG_BASE_DIR="$BK_JOB_STORAGE_BASE_DIR/$KUBERNETES_NAMESPACE/$BK_JOB_POD_NAME/logs" +BK_JOB_LOG_DIR="$BK_JOB_LOG_BASE_DIR/$BK_JOB_APP_NAME" + echo "BK_JOB_LOG_BASE_DIR=$BK_JOB_LOG_BASE_DIR" mkdir -p "$BK_JOB_LOG_BASE_DIR" chmod 777 "$BK_JOB_LOG_BASE_DIR" From 44225a3468b9593d6bbca2fed82f001cf6e226e4 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Mon, 6 Dec 2021 17:28:12 +0800 Subject: [PATCH 04/35] =?UTF-8?q?perf:=20=E5=AE=B9=E5=99=A8=E5=8C=96?= =?UTF-8?q?=E5=90=84=E5=BE=AE=E6=9C=8D=E5=8A=A1=E6=97=A5=E5=BF=97=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E9=87=87=E7=94=A8PVC=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=8F=E8=BF=87=E5=A4=A7=E5=AF=BC=E8=87=B4?= =?UTF-8?q?Pod=E8=A2=AB=E9=A9=B1=E9=80=90=20#507?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chart修复 --- .../charts/bk-job/templates/job-execute/deployment.yaml | 6 +++--- .../bk-job/templates/job-file-gateway/deployment.yaml | 6 +++--- .../bk-job/templates/job-file-worker/statefulset.yaml | 6 +++--- .../charts/bk-job/templates/job-gateway/deployment.yaml | 6 +++--- .../charts/bk-job/templates/job-logsvr/deployment.yaml | 6 +++--- .../charts/bk-job/templates/job-manage/deployment.yaml | 6 +++--- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml index 9cea519e06..c62efa831f 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml @@ -92,9 +92,9 @@ spec: fieldRef: fieldPath: metadata.namespace - name: BK_JOB_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.executeConfig.containerPort }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml index 1feedcdc3c..52bfa0af63 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml @@ -86,9 +86,9 @@ spec: fieldRef: fieldPath: metadata.namespace - name: BK_JOB_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.fileGatewayConfig.containerPort }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml index b18a969f8b..97bcb8d263 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml @@ -97,9 +97,9 @@ spec: fieldRef: fieldPath: metadata.namespace - name: BK_JOB_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.fileWorkerConfig.containerPort }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml index 3f6519c9a6..07f87a0f2a 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml @@ -84,9 +84,9 @@ spec: fieldRef: fieldPath: metadata.namespace - name: BK_JOB_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.gatewayConfig.containerPort.http }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml index dd1834b2c6..cee9d9ffa0 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml @@ -84,9 +84,9 @@ spec: fieldRef: fieldPath: metadata.namespace - name: BK_JOB_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.logsvrConfig.containerPort }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml index 5b86fe97fd..6b39dd55e1 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml @@ -88,9 +88,9 @@ spec: fieldRef: fieldPath: metadata.namespace - name: BK_JOB_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http containerPort: {{ .Values.manageConfig.containerPort }} From b017a21f6931ac247c750c6b718ec3017f5fe339 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Mon, 6 Dec 2021 17:36:38 +0800 Subject: [PATCH 05/35] =?UTF-8?q?perf:=20=E5=AE=B9=E5=99=A8=E5=8C=96?= =?UTF-8?q?=E5=90=84=E5=BE=AE=E6=9C=8D=E5=8A=A1=E6=97=A5=E5=BF=97=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E9=87=87=E7=94=A8PVC=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=8F=E8=BF=87=E5=A4=A7=E5=AF=BC=E8=87=B4?= =?UTF-8?q?Pod=E8=A2=AB=E9=A9=B1=E9=80=90=20#507?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 存储根路径内置与启动脚本 --- .../charts/bk-job/templates/job-analysis/deployment.yaml | 6 ------ .../charts/bk-job/templates/job-backup/deployment.yaml | 4 ---- support-files/kubernetes/images/backend/startup.sh | 2 ++ support-files/kubernetes/images/file-worker/startup.sh | 2 ++ 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml index 787e7960a5..9eb5bff374 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml @@ -71,12 +71,6 @@ spec: value: {{ include "job.profile" . }} - name: BK_JOB_JAR value: job-analysis.jar - - name: BK_JOB_STORAGE_BASE_DIR - value: /data/job/storage - - name: BK_JOB_LOG_BASE_DIR - value: /data/job/storage/logs - - name: BK_JOB_LOG_DIR - value: /data/job/storage/logs/job-analysis - name: JOB_APPLICATION_CONFIGMAP_NAME value: {{ include "common.names.fullname" . }}-analysis - name: JOB_COMMON_CONFIGMAP_NAME diff --git a/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml index 4f01c2a6c6..064cc81887 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml @@ -71,10 +71,6 @@ spec: value: {{ include "job.profile" . }} - name: BK_JOB_JAR value: job-backup.jar - - name: BK_JOB_LOG_BASE_DIR - value: /data/logs/job - - name: BK_JOB_LOG_DIR - value: /data/logs/job/job-backup - name: JOB_APPLICATION_CONFIGMAP_NAME value: {{ include "common.names.fullname" . }}-backup - name: JOB_COMMON_CONFIGMAP_NAME diff --git a/support-files/kubernetes/images/backend/startup.sh b/support-files/kubernetes/images/backend/startup.sh index 2593982d48..bf01d7651e 100644 --- a/support-files/kubernetes/images/backend/startup.sh +++ b/support-files/kubernetes/images/backend/startup.sh @@ -1,5 +1,7 @@ #! /bin/sh +BK_JOB_STORAGE_BASE_DIR="/data/job/storage" + echo "BK_JOB_APP_NAME=$BK_JOB_APP_NAME" echo "KUBERNETES_NAMESPACE=$KUBERNETES_NAMESPACE" echo "BK_JOB_POD_NAME=$BK_JOB_POD_NAME" diff --git a/support-files/kubernetes/images/file-worker/startup.sh b/support-files/kubernetes/images/file-worker/startup.sh index 573ed2bf9a..7f7c05b6dc 100644 --- a/support-files/kubernetes/images/file-worker/startup.sh +++ b/support-files/kubernetes/images/file-worker/startup.sh @@ -1,5 +1,7 @@ #! /bin/sh +BK_JOB_STORAGE_BASE_DIR="/data/job/storage" + echo "BK_JOB_APP_NAME=$BK_JOB_APP_NAME" echo "KUBERNETES_NAMESPACE=$KUBERNETES_NAMESPACE" echo "BK_JOB_POD_NAME=$BK_JOB_POD_NAME" From ede1a0d2e787f9da708a56ac5582681f3bd70ff0 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Mon, 6 Dec 2021 17:58:20 +0800 Subject: [PATCH 06/35] =?UTF-8?q?perf:=20=E5=AE=B9=E5=99=A8=E5=8C=96?= =?UTF-8?q?=E5=90=84=E5=BE=AE=E6=9C=8D=E5=8A=A1=E6=97=A5=E5=BF=97=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E9=87=87=E7=94=A8PVC=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=8F=E8=BF=87=E5=A4=A7=E5=AF=BC=E8=87=B4?= =?UTF-8?q?Pod=E8=A2=AB=E9=A9=B1=E9=80=90=20#507?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kubernetes/charts/bk-job/templates/job-migration/job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support-files/kubernetes/charts/bk-job/templates/job-migration/job.yaml b/support-files/kubernetes/charts/bk-job/templates/job-migration/job.yaml index 961b4cc86c..5fa4bc323e 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-migration/job.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-migration/job.yaml @@ -33,7 +33,7 @@ spec: image: {{ include "job-migration.image" . }} imagePullPolicy: {{ .Values.migration.image.pullPolicy }} command: ["bash"] - args: ["/data/job/startup.sh"] + args: ["/data/job/exec/startup.sh"] env: - name: BK_JOB_MYSQL_HOST value: {{ include "job.mariadb.host" . | quote }} From 12d6d9bcbca07db96f66b7c79727d01dc9d7ba5c Mon Sep 17 00:00:00 2001 From: jsonwan Date: Mon, 6 Dec 2021 20:12:49 +0800 Subject: [PATCH 07/35] =?UTF-8?q?perf:=20=E5=AE=B9=E5=99=A8=E5=8C=96?= =?UTF-8?q?=E5=90=84=E5=BE=AE=E6=9C=8D=E5=8A=A1=E6=97=A5=E5=BF=97=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E9=87=87=E7=94=A8PVC=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=8F=E8=BF=87=E5=A4=A7=E5=AF=BC=E8=87=B4?= =?UTF-8?q?Pod=E8=A2=AB=E9=A9=B1=E9=80=90=20#507?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kubernetes/charts/bk-job/Chart.yaml | 4 +-- .../charts/bk-job/templates/pv-default.yaml | 4 +-- .../charts/bk-job/templates/pvc-storage.yaml | 2 +- .../kubernetes/charts/bk-job/values.yaml | 26 +++++++++++-------- .../kubernetes/images/backend/startup.sh | 2 ++ .../kubernetes/images/file-worker/startup.sh | 1 + 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/support-files/kubernetes/charts/bk-job/Chart.yaml b/support-files/kubernetes/charts/bk-job/Chart.yaml index 8828bc665c..4af6d70736 100644 --- a/support-files/kubernetes/charts/bk-job/Chart.yaml +++ b/support-files/kubernetes/charts/bk-job/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: "bk-job" description: A Helm Chart for Blueking Job type: application -version: 0.1.27 -appVersion: "3.4.2-beta.6" +version: 0.1.28 +appVersion: "3.4.2-beta.7" dependencies: - name: common diff --git a/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml b/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml index b09df791ab..a16fe9dae1 100644 --- a/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml @@ -7,8 +7,8 @@ metadata: spec: storageClassName: manual capacity: - storage: 200Gi + storage: {{ .Values.job.pv.storage }} accessModes: - ReadWriteMany hostPath: - path: "/data/bkjob" + path: {{ .Values.job.pv.path }} diff --git a/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml b/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml index 74d90c5234..1c9f427d78 100644 --- a/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml @@ -8,4 +8,4 @@ spec: - ReadWriteMany resources: requests: - storage: 200Gi + storage: {{ .Values.job.pv.storage }} diff --git a/support-files/kubernetes/charts/bk-job/values.yaml b/support-files/kubernetes/charts/bk-job/values.yaml index 452984105e..bea5180c38 100644 --- a/support-files/kubernetes/charts/bk-job/values.yaml +++ b/support-files/kubernetes/charts/bk-job/values.yaml @@ -491,6 +491,10 @@ job: enabled: true # 各微服务Actuator监控端口 managePort: 19876 + # 存储分配 + pv: + path: "/data/bkjob" + storage: 100Gi ## @section job-gateway parameters gatewayConfig: @@ -500,7 +504,7 @@ gatewayConfig: image: registry: mirrors.tencent.com repository: bkce/job-gateway - tag: 3.4.2-beta.6 + tag: 3.4.2-beta.7 pullPolicy: IfNotPresent pullSecrets: [] replicaCount: 1 @@ -567,7 +571,7 @@ manageConfig: image: registry: mirrors.tencent.com repository: bkce/job-manage - tag: 3.4.2-beta.6 + tag: 3.4.2-beta.7 pullPolicy: IfNotPresent pullSecrets: [] replicaCount: 1 @@ -616,7 +620,7 @@ executeConfig: image: registry: mirrors.tencent.com repository: bkce/job-execute - tag: 3.4.2-beta.6 + tag: 3.4.2-beta.7 pullPolicy: IfNotPresent pullSecrets: [] replicaCount: 1 @@ -662,7 +666,7 @@ crontabConfig: image: registry: mirrors.tencent.com repository: bkce/job-crontab - tag: 3.4.2-beta.6 + tag: 3.4.2-beta.7 pullPolicy: IfNotPresent pullSecrets: [] replicaCount: 1 @@ -716,7 +720,7 @@ logsvrConfig: image: registry: mirrors.tencent.com repository: bkce/job-logsvr - tag: 3.4.2-beta.6 + tag: 3.4.2-beta.7 pullPolicy: IfNotPresent pullSecrets: [] replicaCount: 1 @@ -763,7 +767,7 @@ backupConfig: image: registry: mirrors.tencent.com repository: bkce/job-backup - tag: 3.4.2-beta.6 + tag: 3.4.2-beta.7 pullPolicy: IfNotPresent pullSecrets: [] replicaCount: 1 @@ -807,7 +811,7 @@ analysisConfig: image: registry: mirrors.tencent.com repository: bkce/job-analysis - tag: 3.4.2-beta.6 + tag: 3.4.2-beta.7 pullPolicy: IfNotPresent pullSecrets: [] replicaCount: 1 @@ -851,7 +855,7 @@ fileGatewayConfig: image: registry: mirrors.tencent.com repository: bkce/job-file-gateway - tag: 3.4.2-beta.6 + tag: 3.4.2-beta.7 pullPolicy: IfNotPresent pullSecrets: [] replicaCount: 1 @@ -895,7 +899,7 @@ fileWorkerConfig: image: registry: mirrors.tencent.com repository: bkce/job-file-worker - tag: 3.4.2-beta.6 + tag: 3.4.2-beta.7 pullPolicy: IfNotPresent pullSecrets: [] replicaCount: 1 @@ -951,7 +955,7 @@ frontendConfig: image: registry: mirrors.tencent.com repository: bkce/job-frontend - tag: 3.4.2-beta.6 + tag: 3.4.2-beta.7 pullPolicy: IfNotPresent pullSecrets: [] replicaCount: 1 @@ -1011,5 +1015,5 @@ migration: image: registry: "mirrors.tencent.com" repository: "bkce/job-migration" - tag: 3.4.2-beta.6 + tag: 3.4.2-beta.7 pullPolicy: IfNotPresent diff --git a/support-files/kubernetes/images/backend/startup.sh b/support-files/kubernetes/images/backend/startup.sh index bf01d7651e..42ff75fbe5 100644 --- a/support-files/kubernetes/images/backend/startup.sh +++ b/support-files/kubernetes/images/backend/startup.sh @@ -19,6 +19,8 @@ echo "BK_JOB_LOG_DIR=$BK_JOB_LOG_DIR" mkdir -p "$BK_JOB_LOG_DIR" chmod 777 "$BK_JOB_LOG_DIR" +# 创建软链接供容器内日志采集使用 +ln -s $BK_JOB_LOG_BASE_DIR /data/logs ls $BK_JOB_LOG_BASE_DIR java -server \ diff --git a/support-files/kubernetes/images/file-worker/startup.sh b/support-files/kubernetes/images/file-worker/startup.sh index 7f7c05b6dc..a40a02e926 100644 --- a/support-files/kubernetes/images/file-worker/startup.sh +++ b/support-files/kubernetes/images/file-worker/startup.sh @@ -25,6 +25,7 @@ chmod 777 "$BK_JOB_FILE_WORKER_WORKSPACE_DIR" echo "BK_JOB_NODE_IP=$BK_JOB_NODE_IP" +ln -s $BK_JOB_LOG_BASE_DIR /data/logs ls $BK_JOB_LOG_BASE_DIR java -server \ From 462c6105856b9459808f23abe902598c23a77133 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Mon, 6 Dec 2021 20:26:57 +0800 Subject: [PATCH 08/35] =?UTF-8?q?perf:=20=E5=AE=B9=E5=99=A8=E5=8C=96?= =?UTF-8?q?=E5=90=84=E5=BE=AE=E6=9C=8D=E5=8A=A1=E6=97=A5=E5=BF=97=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E9=87=87=E7=94=A8PVC=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=8F=E8=BF=87=E5=A4=A7=E5=AF=BC=E8=87=B4?= =?UTF-8?q?Pod=E8=A2=AB=E9=A9=B1=E9=80=90=20#507?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- support-files/kubernetes/charts/bk-job/templates/bklog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support-files/kubernetes/charts/bk-job/templates/bklog.yaml b/support-files/kubernetes/charts/bk-job/templates/bklog.yaml index 17265c9354..ad2191a846 100644 --- a/support-files/kubernetes/charts/bk-job/templates/bklog.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/bklog.yaml @@ -14,7 +14,7 @@ spec: app.kubernetes.io/name: {{ include "common.names.fullname" . }} {{ include "job.labelKeys.jobScope" . }}: {{ include "job.labelValues.jobScope.backend" . }} path: - - /data/logs/job/job-*/*.log + - /data/logs/job-*/*.log encoding: 'utf-8' multiline: pattern: '^\[(?P.*?)\]\s+?' From 86ea444610f534b959c26005a70e23913b4b78e9 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Wed, 8 Dec 2021 10:53:38 +0800 Subject: [PATCH 09/35] =?UTF-8?q?perf:=20=E5=AE=B9=E5=99=A8=E5=8C=96?= =?UTF-8?q?=E5=90=84=E5=BE=AE=E6=9C=8D=E5=8A=A1=E6=97=A5=E5=BF=97=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E9=87=87=E7=94=A8PVC=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=8F=E8=BF=87=E5=A4=A7=E5=AF=BC=E8=87=B4?= =?UTF-8?q?Pod=E8=A2=AB=E9=A9=B1=E9=80=90=20#507?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../charts/bk-job/templates/job-crontab/deployment.yaml | 4 ---- .../charts/bk-job/templates/job-execute/deployment.yaml | 4 ---- .../charts/bk-job/templates/job-file-gateway/deployment.yaml | 4 ---- .../charts/bk-job/templates/job-file-worker/statefulset.yaml | 4 ---- .../charts/bk-job/templates/job-gateway/deployment.yaml | 4 ---- .../charts/bk-job/templates/job-logsvr/deployment.yaml | 4 ---- .../charts/bk-job/templates/job-manage/deployment.yaml | 4 ---- 7 files changed, 28 deletions(-) diff --git a/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml index f493a53fb2..e0e68eeb2f 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml @@ -72,10 +72,6 @@ spec: value: {{ include "job.profile" . }} - name: BK_JOB_JAR value: job-crontab.jar - - name: BK_JOB_LOG_BASE_DIR - value: /data/logs/job - - name: BK_JOB_LOG_DIR - value: /data/logs/job/job-crontab - name: JOB_APPLICATION_CONFIGMAP_NAME value: {{ include "common.names.fullname" . }}-crontab - name: JOB_COMMON_CONFIGMAP_NAME diff --git a/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml index c62efa831f..23868cf81c 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml @@ -75,10 +75,6 @@ spec: valueFrom: fieldRef: fieldPath: status.hostIP - - name: BK_JOB_LOG_BASE_DIR - value: /data/logs/job - - name: BK_JOB_LOG_DIR - value: /data/logs/job/job-execute - name: BK_JOB_STORAGE_ROOT_PATH value: {{ .Values.job.storage.rootPath }} - name: JOB_APPLICATION_CONFIGMAP_NAME diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml index 52bfa0af63..168d0152ad 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml @@ -71,10 +71,6 @@ spec: value: {{ include "job.profile" . }} - name: BK_JOB_JAR value: job-file-gateway.jar - - name: BK_JOB_LOG_BASE_DIR - value: /data/logs/job - - name: BK_JOB_LOG_DIR - value: /data/logs/job/job-file-gateway - name: JOB_APPLICATION_CONFIGMAP_NAME value: {{ include "common.names.fullname" . }}-file-gateway - name: JOB_COMMON_CONFIGMAP_NAME diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml index 97bcb8d263..ec5b4874ad 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml @@ -70,10 +70,6 @@ spec: value: {{ include "job.profile" . }} - name: BK_JOB_JAR value: job-file-worker.jar - - name: BK_JOB_LOG_BASE_DIR - value: /data/logs/job - - name: BK_JOB_LOG_DIR - value: /data/logs/job/job-file-worker - name: BK_JOB_FILE_WORKER_WORKSPACE_DIR value: {{ .Values.fileWorkerConfig.downloadFile.dir }}/JobFileWorkerWorkspace - name: BK_JOB_NODE_IP diff --git a/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml index 07f87a0f2a..010a9a7361 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml @@ -69,10 +69,6 @@ spec: value: {{ include "job.profile" . }} - name: BK_JOB_JAR value: job-gateway.jar - - name: BK_JOB_LOG_BASE_DIR - value: /data/logs/job - - name: BK_JOB_LOG_DIR - value: /data/logs/job/job-gateway - name: JOB_APPLICATION_CONFIGMAP_NAME value: {{ include "common.names.fullname" . }}-gateway - name: JOB_COMMON_CONFIGMAP_NAME diff --git a/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml index cee9d9ffa0..365f7e2312 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml @@ -69,10 +69,6 @@ spec: value: {{ include "job.profile" . }} - name: BK_JOB_JAR value: job-logsvr.jar - - name: BK_JOB_LOG_BASE_DIR - value: /data/logs/job - - name: BK_JOB_LOG_DIR - value: /data/logs/job/job-logsvr - name: JOB_APPLICATION_CONFIGMAP_NAME value: {{ include "common.names.fullname" . }}-logsvr - name: JOB_COMMON_CONFIGMAP_NAME diff --git a/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml index 6b39dd55e1..516929130a 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml @@ -73,10 +73,6 @@ spec: value: {{ include "job.profile" . }} - name: BK_JOB_JAR value: job-manage.jar - - name: BK_JOB_LOG_BASE_DIR - value: "/data/logs/job" - - name: BK_JOB_LOG_DIR - value: "/data/logs/job/job-manage" - name: JOB_APPLICATION_CONFIGMAP_NAME value: {{ include "common.names.fullname" . }}-manage - name: JOB_COMMON_CONFIGMAP_NAME From a71914d7ddfedae0491de36a6f7e7efdd3d832f8 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Thu, 9 Dec 2021 11:39:42 +0800 Subject: [PATCH 10/35] =?UTF-8?q?fix:=20=E5=88=86=E5=8F=91=E7=AC=AC?= =?UTF-8?q?=E4=B8=89=E6=96=B9=E6=BA=90=E6=96=87=E4=BB=B6=E5=81=B6=E7=8E=B0?= =?UTF-8?q?=E5=B0=8F=E6=96=87=E4=BB=B6MD5=E4=B8=8D=E6=AD=A3=E7=A1=AE=20#52?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../artifactory/sdk/ArtifactoryClient.java | 7 ++--- .../tencent/bk/job/common/util/FileUtil.java | 27 +++++++++++-------- .../api/web/impl/WebBackupResourceImpl.java | 3 ++- .../backup/executor/ImportJobExecutor.java | 4 ++- .../api/web/impl/WebTaskLogResourceImpl.java | 3 ++- .../ArtifactoryLocalFilePrepareTask.java | 10 +++++-- .../worker/cos/service/DownloadFileTask.java | 11 ++++++-- .../file/worker/cos/service/RemoteClient.java | 3 ++- .../service/ArtifactoryRemoteClient.java | 3 ++- .../worker/cos/JobTencentInnerCOSClient.java | 3 ++- .../file/worker/cos/TencentInnerCOSUtil.java | 7 ++--- .../worker/cos/service/COSRemoteClient.java | 3 ++- 12 files changed, 56 insertions(+), 28 deletions(-) diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java index 928fb1341f..23cee5ec15 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java @@ -61,6 +61,7 @@ import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; @@ -385,12 +386,12 @@ public NodeDTO getFileNode(String filePath) { return nodeDTO; } - public InputStream getFileInputStream(String filePath) throws ServiceException { + public Pair getFileInputStream(String filePath) throws ServiceException { List pathList = parsePath(filePath); return getFileInputStream(pathList.get(0), pathList.get(1), pathList.get(2)); } - public InputStream getFileInputStream(String projectId, String repoName, String filePath) throws ServiceException { + public Pair getFileInputStream(String projectId, String repoName, String filePath) throws ServiceException { DownloadGenericFileReq req = new DownloadGenericFileReq(); req.setProject(projectId); req.setRepo(repoName); @@ -400,7 +401,7 @@ public InputStream getFileInputStream(String projectId, String repoName, String CloseableHttpResponse resp; try { resp = longHttpHelper.getRawResp(false, url, getJsonHeaders()); - return resp.getEntity().getContent(); + return Pair.of(resp.getEntity().getContent(), resp.getEntity().getContentLength()); } catch (IOException e) { log.error("Fail to getFileInputStream", e); throw new InternalException(ErrorCode.FAIL_TO_REQUEST_THIRD_FILE_SOURCE_DOWNLOAD_GENERIC_FILE); diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FileUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FileUtil.java index 9259f233bc..ca0229e851 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FileUtil.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FileUtil.java @@ -77,6 +77,7 @@ public static String writeInsToFile(InputStream ins, String targetPath) throws I fos.write(content, 0, length); Thread.sleep(0); } + closeFos(fos); fis = new FileInputStream(targetPath); return DigestUtils.md5Hex(fis); } catch (FileNotFoundException e) { @@ -89,7 +90,7 @@ public static String writeInsToFile(InputStream ins, String targetPath) throws I log.info("Download interrupted, targetPath:{}", targetPath); throw e; } finally { - closeStreams(ins, fos, fis); + closeStreams(ins, fis); } } @@ -139,6 +140,8 @@ public static String writeInsToFile(InputStream ins, String targetPath, Long fil } Thread.sleep(0); } + closeFos(fos); + log.info("targetPath:{},totalLength={},fileSize={}", targetPath, totalLength, fileSize); currentSpeedWatchTime = System.currentTimeMillis(); timeDelta = currentSpeedWatchTime - lastSpeedWatchTime; long fileSizeDelta = totalLength - lastSpeedWatchFileSize; @@ -158,13 +161,23 @@ public static String writeInsToFile(InputStream ins, String targetPath, Long fil log.info("Download interrupted, targetPath:{}", targetPath); throw e; } finally { - closeStreams(ins, fos, fis); + closeStreams(ins, fis); + } + } + + private static void closeFos(FileOutputStream fos) { + if (fos != null) { + try { + fos.flush(); + fos.close(); + } catch (IOException e) { + log.error("Fail to close fos", e); + } } } private static void closeStreams( InputStream ins, - FileOutputStream fos, FileInputStream fis ) { if (fis != null) { @@ -181,14 +194,6 @@ private static void closeStreams( log.error("Fail to close ins", e); } } - if (fos != null) { - try { - fos.flush(); - fos.close(); - } catch (IOException e) { - log.error("Fail to close fos", e); - } - } } /** diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebBackupResourceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebBackupResourceImpl.java index c22498606e..8a64d14124 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebBackupResourceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebBackupResourceImpl.java @@ -184,11 +184,12 @@ private Pair getFileSizeAndStreamFromArtifactory(St throw new InternalException(ErrorCode.FAIL_TO_GET_NODE_INFO_FROM_ARTIFACTORY); } try { - ins = artifactoryClient.getFileInputStream( + Pair pair = artifactoryClient.getFileInputStream( artifactoryConfig.getArtifactoryJobProject(), backupStorageConfig.getBackupRepo(), fileName ); + ins = pair.getLeft(); } catch (Exception e) { throw new InternalException(ErrorCode.FAIL_TO_DOWNLOAD_NODE_FROM_ARTIFACTORY); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java index 7d66dfa0c8..d48538ef83 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java @@ -72,6 +72,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.stereotype.Service; @@ -201,11 +202,12 @@ private void processImportJob(String jobId) { // 下载文件 if (!importFileDirectory.exists()) { log.debug("begin to download from artifactory:{}", importJob.getFileName()); - InputStream ins = artifactoryClient.getFileInputStream( + Pair pair = artifactoryClient.getFileInputStream( artifactoryConfig.getArtifactoryJobProject(), backupStorageConfig.getBackupRepo(), importJob.getFileName() ); + InputStream ins = pair.getLeft(); String localPath = PathUtil.joinFilePath( storageService.getStoragePath(), importJob.getFileName() diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java index ef1176d897..f9f4dd7da3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java @@ -218,11 +218,12 @@ private Pair getFileSizeAndStreamFromArtifactory( } try { log.debug("get {} fileInputStream from artifactory", exportInfo.getZipFileName()); - ins = artifactoryClient.getFileInputStream( + Pair pair = artifactoryClient.getFileInputStream( artifactoryConfig.getArtifactoryJobProject(), logExportConfig.getLogExportRepo(), exportInfo.getZipFileName() ); + ins = pair.getLeft(); } catch (Exception e) { throw new InternalException(ErrorCode.FAIL_TO_DOWNLOAD_NODE_FROM_ARTIFACTORY); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java index c2020275f0..1fd008a3b0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java @@ -37,6 +37,7 @@ import com.tencent.bk.job.logsvr.model.service.ServiceIpLogDTO; import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import java.io.File; import java.io.InputStream; @@ -217,13 +218,18 @@ public Void call() throws Exception { // 制品库的完整路径 String fullFilePath = PathUtil.joinFilePath(jobLocalUploadRootPath, filePath); NodeDTO nodeDTO = artifactoryClient.getFileNode(fullFilePath); - InputStream ins = artifactoryClient.getFileInputStream(fullFilePath); + Pair pair = artifactoryClient.getFileInputStream(fullFilePath); + InputStream ins = pair.getLeft(); + long length = pair.getRight(); + if (nodeDTO.getSize() != length) { + log.warn("{},ins length={},node.size={}", filePath, length, nodeDTO.getSize()); + } // 保存到本地临时目录 AtomicInteger speed = new AtomicInteger(0); AtomicInteger process = new AtomicInteger(0); try { log.debug("Download {} to {}", fullFilePath, localPath); - FileUtil.writeInsToFile(ins, localPath, nodeDTO.getSize(), speed, process); + FileUtil.writeInsToFile(ins, localPath, length, speed, process); } catch (InterruptedException e) { log.warn("Interrupted:Download {} to {}", fullFilePath, localPath); } catch (Exception e) { diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java index 7baee5a982..cab1e7899e 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java @@ -32,6 +32,7 @@ import com.tencent.bk.job.file.worker.model.FileMetaData; import com.tencent.bk.job.file_gateway.consts.TaskCommandEnum; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import java.io.File; import java.io.IOException; @@ -53,6 +54,7 @@ class DownloadFileTask extends Thread { TaskReporter taskReporter; DownloadFileTaskEventListener taskEventListener; FileProgressWatchingTask watchingTask; + public DownloadFileTask(RemoteClient remoteClient, String taskId, String filePath, String downloadFileDir, String filePrefix, AtomicLong fileSize, AtomicInteger speed, AtomicInteger process, FileProgressWatchingTask watchingTask, @@ -100,8 +102,13 @@ public void downloadFileToLocal( long fileSize = metaData.getSize(); fileMd5 = metaData.getMd5(); fileSizeWrapper.set(fileSize); - ins = remoteClient.getFileInputStream(filePath); - currentMd5 = FileUtil.writeInsToFile(ins, targetPath, fileSize, speed, process); + Pair pair = remoteClient.getFileInputStream(filePath); + ins = pair.getLeft(); + long length = pair.getRight(); + if (length != fileSize) { + log.warn("{},fileSize={},ins length={}", filePath, fileSize, length); + } + currentMd5 = FileUtil.writeInsToFile(ins, targetPath, length, speed, process); if (fileMd5 == null) { log.warn("No Md5 in metadata, do not check,key={},targetPath={},fileMd5={},currentMd5={}", filePath, targetPath, fileMd5, currentMd5); diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/RemoteClient.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/RemoteClient.java index 1d9f650eb1..a9dbb020b7 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/RemoteClient.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/RemoteClient.java @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.file.worker.model.FileMetaData; +import org.apache.commons.lang3.tuple.Pair; import java.io.InputStream; @@ -33,7 +34,7 @@ public interface RemoteClient { FileMetaData getFileMetaData(String filePath); - InputStream getFileInputStream(String filePath) throws ServiceException; + Pair getFileInputStream(String filePath) throws ServiceException; void shutdown(); diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java index d530942e71..b0d92b3c4f 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java @@ -31,6 +31,7 @@ import com.tencent.bk.job.file.worker.model.FileMetaData; import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import java.io.InputStream; @@ -48,7 +49,7 @@ public FileMetaData getFileMetaData(String filePath) { } @Override - public InputStream getFileInputStream(String filePath) throws ServiceException { + public Pair getFileInputStream(String filePath) throws ServiceException { return super.getFileInputStream(filePath); } diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java index a80b326bff..a4efc7ddca 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java @@ -28,6 +28,7 @@ import com.tencent.cos.model.Bucket; import com.tencent.cos.model.COSObjectSummary; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import java.io.File; import java.io.InputStream; @@ -105,7 +106,7 @@ public FileMetaData getFileMetaData(String bucketName, String key) { key); } - public InputStream getFileInputStream(String bucketName, String key) { + public Pair getFileInputStream(String bucketName, String key) { return TencentInnerCOSUtil.getFileInputStream(accessKey, secretKey, regionName, getRealBucketName(bucketName) , key); } diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/TencentInnerCOSUtil.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/TencentInnerCOSUtil.java index 9afcfcc6d1..170527fb59 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/TencentInnerCOSUtil.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/TencentInnerCOSUtil.java @@ -44,6 +44,7 @@ import com.tencent.cos.region.Region; import com.tencent.cos.utils.DateUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import java.io.BufferedOutputStream; import java.io.File; @@ -136,15 +137,15 @@ public static List listAllObjects( } } - public static InputStream getFileInputStream(String accessKey, String secretKey, String regionName, - String bucketName, String key) { + public static Pair getFileInputStream(String accessKey, String secretKey, String regionName, + String bucketName, String key) { COSClient cosClient = getCOSClient(accessKey, secretKey, regionName); COSObject cosObject = cosClient.getObject(bucketName, key); if (cosObject == null) { throw new InternalException(ErrorCode.FAIL_TO_REQUEST_THIRD_FILE_SOURCE_DOWNLOAD_GENERIC_FILE, new String[]{String.format("Fail to getObject by bucketName %s key %s", bucketName, key)}); } - return cosObject.getObjectContent(); + return Pair.of(cosObject.getObjectContent(), cosObject.getObjectMetadata().getContentLength()); } public static FileMetaData getFileMetaData(String accessKey, String secretKey, String regionName, diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSRemoteClient.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSRemoteClient.java index e903a9e57f..5fa5cb0198 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSRemoteClient.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSRemoteClient.java @@ -26,6 +26,7 @@ import com.tencent.bk.job.file.worker.cos.JobTencentInnerCOSClient; import com.tencent.bk.job.file.worker.model.FileMetaData; +import org.apache.commons.lang3.tuple.Pair; import java.io.InputStream; import java.util.ArrayList; @@ -56,7 +57,7 @@ public FileMetaData getFileMetaData(String filePath) { } @Override - public InputStream getFileInputStream(String filePath) { + public Pair getFileInputStream(String filePath) { List pathList = parsePath(filePath); return jobTencentInnerCOSClient.getFileInputStream(pathList.get(0), pathList.get(1)); } From b2913255813b139e6fc1b2d8a26ee6bcfe4661fc Mon Sep 17 00:00:00 2001 From: jsonwan Date: Thu, 9 Dec 2021 12:24:00 +0800 Subject: [PATCH 11/35] =?UTF-8?q?fix:=20=E5=88=86=E5=8F=91=E7=AC=AC?= =?UTF-8?q?=E4=B8=89=E6=96=B9=E6=BA=90=E6=96=87=E4=BB=B6=E5=81=B6=E7=8E=B0?= =?UTF-8?q?=E5=B0=8F=E6=96=87=E4=BB=B6MD5=E4=B8=8D=E6=AD=A3=E7=A1=AE=20#52?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bk/job/common/artifactory/sdk/ArtifactoryClient.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java index 23cee5ec15..5625c39df3 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java @@ -401,7 +401,8 @@ public Pair getFileInputStream(String projectId, String repoN CloseableHttpResponse resp; try { resp = longHttpHelper.getRawResp(false, url, getJsonHeaders()); - return Pair.of(resp.getEntity().getContent(), resp.getEntity().getContentLength()); + return Pair.of(resp.getEntity().getContent(), + Long.parseLong(resp.getFirstHeader("Content-Length").getValue())); } catch (IOException e) { log.error("Fail to getFileInputStream", e); throw new InternalException(ErrorCode.FAIL_TO_REQUEST_THIRD_FILE_SOURCE_DOWNLOAD_GENERIC_FILE); From 198beed67aab80e0d73d29fd851146d1d9e7951a Mon Sep 17 00:00:00 2001 From: jsonwan Date: Thu, 9 Dec 2021 22:11:03 +0800 Subject: [PATCH 12/35] =?UTF-8?q?chore:=20=E5=A2=9E=E5=8A=A0preCI=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=20#523?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9e9a9f0926..72e96c54e4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,6 @@ out/ ### application ### **/*.log support-files/dependJarLists/ -**/*.DS_Store \ No newline at end of file +**/*.DS_Store +.codecc +build.yml From c074e748d16c650cfa18b1a20cfd2fb27aa6326b Mon Sep 17 00:00:00 2001 From: jsonwan Date: Mon, 13 Dec 2021 17:56:28 +0800 Subject: [PATCH 13/35] =?UTF-8?q?perf:=20=E5=AE=B9=E5=99=A8=E5=8C=96?= =?UTF-8?q?=E5=90=84=E5=BE=AE=E6=9C=8D=E5=8A=A1=E6=97=A5=E5=BF=97=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E9=87=87=E7=94=A8PVC=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=8F=E8=BF=87=E5=A4=A7=E5=AF=BC=E8=87=B4?= =?UTF-8?q?Pod=E8=A2=AB=E9=A9=B1=E9=80=90=20#507?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chart规范化 --- .../bk-job/templates/job-analysis/deployment.yaml | 2 +- .../charts/bk-job/templates/job-backup/deployment.yaml | 2 +- .../bk-job/templates/job-crontab/deployment.yaml | 2 +- .../bk-job/templates/job-execute/deployment.yaml | 2 +- .../bk-job/templates/job-file-gateway/deployment.yaml | 2 +- .../bk-job/templates/job-file-worker/statefulset.yaml | 2 +- .../bk-job/templates/job-gateway/deployment.yaml | 2 +- .../charts/bk-job/templates/job-logsvr/deployment.yaml | 2 +- .../charts/bk-job/templates/job-manage/deployment.yaml | 2 +- .../kubernetes/charts/bk-job/templates/pv-default.yaml | 9 ++++++++- .../charts/bk-job/templates/pvc-storage.yaml | 10 +++++++++- 11 files changed, 26 insertions(+), 11 deletions(-) diff --git a/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml index 9eb5bff374..1c4351516f 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-analysis/deployment.yaml @@ -124,7 +124,7 @@ spec: volumes: - name: job-storage persistentVolumeClaim: - claimName: bk-job-pv-claim + claimName: {{ include "common.names.fullname" . }}-pv-claim - name: mariadb secret: secretName: {{ include "job.mariadb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml index 064cc81887..2917cdbe06 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-backup/deployment.yaml @@ -124,7 +124,7 @@ spec: volumes: - name: job-storage persistentVolumeClaim: - claimName: bk-job-pv-claim + claimName: {{ include "common.names.fullname" . }}-pv-claim - name: mariadb secret: secretName: {{ include "job.mariadb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml index e0e68eeb2f..64a3eba512 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-crontab/deployment.yaml @@ -125,7 +125,7 @@ spec: volumes: - name: job-storage persistentVolumeClaim: - claimName: bk-job-pv-claim + claimName: {{ include "common.names.fullname" . }}-pv-claim - name: mariadb secret: secretName: {{ include "job.mariadb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml index 23868cf81c..705ad1e6d9 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-execute/deployment.yaml @@ -138,7 +138,7 @@ spec: volumes: - name: job-storage persistentVolumeClaim: - claimName: bk-job-pv-claim + claimName: {{ include "common.names.fullname" . }}-pv-claim - name: local-file-dir hostPath: path: {{ .Values.job.storage.rootPath }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml index 168d0152ad..ed4e983874 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-file-gateway/deployment.yaml @@ -124,7 +124,7 @@ spec: volumes: - name: job-storage persistentVolumeClaim: - claimName: bk-job-pv-claim + claimName: {{ include "common.names.fullname" . }}-pv-claim - name: mariadb secret: secretName: {{ include "job.mariadb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml b/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml index ec5b4874ad..23ed28db7f 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-file-worker/statefulset.yaml @@ -128,7 +128,7 @@ spec: volumes: - name: job-storage persistentVolumeClaim: - claimName: bk-job-pv-claim + claimName: {{ include "common.names.fullname" . }}-pv-claim - name: workspace hostPath: path: {{ .Values.fileWorkerConfig.downloadFile.dir }}/JobFileWorkerWorkspace diff --git a/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml index 010a9a7361..e19a7bf9cb 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-gateway/deployment.yaml @@ -125,7 +125,7 @@ spec: volumes: - name: job-storage persistentVolumeClaim: - claimName: bk-job-pv-claim + claimName: {{ include "common.names.fullname" . }}-pv-claim - name: job-gateway-tls-certs secret: secretName: {{ printf "%s-gateway-%s" (include "common.names.fullname" .) "tls-cert" }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml index 365f7e2312..9ab4b5498d 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-logsvr/deployment.yaml @@ -116,7 +116,7 @@ spec: volumes: - name: job-storage persistentVolumeClaim: - claimName: bk-job-pv-claim + claimName: {{ include "common.names.fullname" . }}-pv-claim - name: mongodb secret: secretName: {{ include "job.mongodb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml b/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml index 516929130a..2254090f47 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-manage/deployment.yaml @@ -132,7 +132,7 @@ spec: volumes: - name: job-storage persistentVolumeClaim: - claimName: bk-job-pv-claim + claimName: {{ include "common.names.fullname" . }}-pv-claim - name: mariadb secret: secretName: {{ include "job.mariadb.secretName" . }} diff --git a/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml b/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml index a16fe9dae1..7c17ee42b9 100644 --- a/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/pv-default.yaml @@ -1,9 +1,16 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: bk-job-local-volume + name: {{ include "common.names.fullname" . }}-local labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} type: local + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} spec: storageClassName: manual capacity: diff --git a/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml b/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml index 1c9f427d78..8c36b8aba6 100644 --- a/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/pvc-storage.yaml @@ -1,7 +1,15 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: bk-job-pv-claim + name: {{ include "common.names.fullname" . }}-pv-claim + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} spec: storageClassName: manual accessModes: From 280e184d8a79c184edfe6c172170aeecee3f1c44 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Tue, 14 Dec 2021 19:54:33 +0800 Subject: [PATCH 14/35] =?UTF-8?q?fix:=20=E5=88=86=E5=8F=91=E7=AC=AC?= =?UTF-8?q?=E4=B8=89=E6=96=B9=E6=BA=90=E6=96=87=E4=BB=B6=E5=81=B6=E7=8E=B0?= =?UTF-8?q?=E5=B0=8F=E6=96=87=E4=BB=B6MD5=E4=B8=8D=E6=AD=A3=E7=A1=AE=20#52?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../job/common/artifactory/sdk/ArtifactoryClient.java | 11 +++++++++-- .../local/ArtifactoryLocalFilePrepareTask.java | 10 +++++++--- .../job/file/worker/cos/service/DownloadFileTask.java | 2 +- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java index 5625c39df3..8cc29796df 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java @@ -401,8 +401,15 @@ public Pair getFileInputStream(String projectId, String repoN CloseableHttpResponse resp; try { resp = longHttpHelper.getRawResp(false, url, getJsonHeaders()); - return Pair.of(resp.getEntity().getContent(), - Long.parseLong(resp.getFirstHeader("Content-Length").getValue())); + Header contentLengthHeader = resp.getFirstHeader("Content-Length"); + Long contentLength = null; + if (contentLengthHeader != null && StringUtils.isNotBlank(contentLengthHeader.getValue())) { + contentLength = Long.parseLong(contentLengthHeader.getValue()); + log.debug("Content-Length from header:{}", contentLengthHeader.getValue()); + } else { + log.debug("Content-Length from header is null or blank"); + } + return Pair.of(resp.getEntity().getContent(), contentLength); } catch (IOException e) { log.error("Fail to getFileInputStream", e); throw new InternalException(ErrorCode.FAIL_TO_REQUEST_THIRD_FILE_SOURCE_DOWNLOAD_GENERIC_FILE); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java index 1fd008a3b0..8572f5a904 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java @@ -220,16 +220,20 @@ public Void call() throws Exception { NodeDTO nodeDTO = artifactoryClient.getFileNode(fullFilePath); Pair pair = artifactoryClient.getFileInputStream(fullFilePath); InputStream ins = pair.getLeft(); - long length = pair.getRight(); - if (nodeDTO.getSize() != length) { + Long length = pair.getRight(); + Long fileSize = nodeDTO.getSize(); + if (fileSize != null && !fileSize.equals(length)) { log.warn("{},ins length={},node.size={}", filePath, length, nodeDTO.getSize()); } + if (fileSize == null || fileSize <= 0) { + fileSize = length; + } // 保存到本地临时目录 AtomicInteger speed = new AtomicInteger(0); AtomicInteger process = new AtomicInteger(0); try { log.debug("Download {} to {}", fullFilePath, localPath); - FileUtil.writeInsToFile(ins, localPath, length, speed, process); + FileUtil.writeInsToFile(ins, localPath, fileSize, speed, process); } catch (InterruptedException e) { log.warn("Interrupted:Download {} to {}", fullFilePath, localPath); } catch (Exception e) { diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java index cab1e7899e..b5ec6b9ba7 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java @@ -108,7 +108,7 @@ public void downloadFileToLocal( if (length != fileSize) { log.warn("{},fileSize={},ins length={}", filePath, fileSize, length); } - currentMd5 = FileUtil.writeInsToFile(ins, targetPath, length, speed, process); + currentMd5 = FileUtil.writeInsToFile(ins, targetPath, fileSize, speed, process); if (fileMd5 == null) { log.warn("No Md5 in metadata, do not check,key={},targetPath={},fileMd5={},currentMd5={}", filePath, targetPath, fileMd5, currentMd5); From f8c439a08bacda3d7b086af6daf15200499770b4 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Tue, 14 Dec 2021 20:03:14 +0800 Subject: [PATCH 15/35] =?UTF-8?q?fix:=20=E5=88=86=E5=8F=91=E7=AC=AC?= =?UTF-8?q?=E4=B8=89=E6=96=B9=E6=BA=90=E6=96=87=E4=BB=B6=E5=81=B6=E7=8E=B0?= =?UTF-8?q?=E5=B0=8F=E6=96=87=E4=BB=B6MD5=E4=B8=8D=E6=AD=A3=E7=A1=AE=20#52?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit format --- .../bk/job/file/worker/cos/JobTencentInnerCOSClient.java | 2 +- .../tencent/bk/job/file/worker/cos/service/COSRemoteClient.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java index a4efc7ddca..4395677e45 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java @@ -106,7 +106,7 @@ public FileMetaData getFileMetaData(String bucketName, String key) { key); } - public Pair getFileInputStream(String bucketName, String key) { + public Pair getFileInputStream(String bucketName, String key) { return TencentInnerCOSUtil.getFileInputStream(accessKey, secretKey, regionName, getRealBucketName(bucketName) , key); } diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSRemoteClient.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSRemoteClient.java index 5fa5cb0198..ecc7503846 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSRemoteClient.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSRemoteClient.java @@ -57,7 +57,7 @@ public FileMetaData getFileMetaData(String filePath) { } @Override - public Pair getFileInputStream(String filePath) { + public Pair getFileInputStream(String filePath) { List pathList = parsePath(filePath); return jobTencentInnerCOSClient.getFileInputStream(pathList.get(0), pathList.get(1)); } From 022627a4e3dbd3ed75063378959ba4a3afad76fd Mon Sep 17 00:00:00 2001 From: jsonwan Date: Tue, 14 Dec 2021 20:05:39 +0800 Subject: [PATCH 16/35] =?UTF-8?q?fix:=20=E5=88=86=E5=8F=91=E7=AC=AC?= =?UTF-8?q?=E4=B8=89=E6=96=B9=E6=BA=90=E6=96=87=E4=BB=B6=E5=81=B6=E7=8E=B0?= =?UTF-8?q?=E5=B0=8F=E6=96=87=E4=BB=B6MD5=E4=B8=8D=E6=AD=A3=E7=A1=AE=20#52?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit format --- .../tencent/bk/job/file/worker/cos/service/RemoteClient.java | 2 +- .../worker/artifactory/service/ArtifactoryRemoteClient.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/RemoteClient.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/RemoteClient.java index a9dbb020b7..99e1d7d24c 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/RemoteClient.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/RemoteClient.java @@ -34,7 +34,7 @@ public interface RemoteClient { FileMetaData getFileMetaData(String filePath); - Pair getFileInputStream(String filePath) throws ServiceException; + Pair getFileInputStream(String filePath) throws ServiceException; void shutdown(); diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java index b0d92b3c4f..ba35eb0ad2 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java @@ -49,7 +49,7 @@ public FileMetaData getFileMetaData(String filePath) { } @Override - public Pair getFileInputStream(String filePath) throws ServiceException { + public Pair getFileInputStream(String filePath) throws ServiceException { return super.getFileInputStream(filePath); } From ed53a7d144919a8623bcf5e9c44aca7462f71ffa Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 15 Dec 2021 08:05:48 +0800 Subject: [PATCH 17/35] =?UTF-8?q?bugfix:=20=E7=94=A8=E6=88=B7=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E6=96=B9=E6=A1=88=E4=B8=AD=E7=9A=84=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E8=AE=BE=E7=BD=AE=E4=B8=BA0=EF=BC=8C?= =?UTF-8?q?=E4=BD=9C=E4=B8=9A=E5=9C=A8=E6=89=A7=E8=A1=8C5min=E5=90=8E?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E5=A4=B1=E8=B4=A5=20#532?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/TaskExecuteServiceImpl.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index e724ed15d7..fd28c5c58a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -59,6 +59,7 @@ import com.tencent.bk.job.execute.constants.UserOperationEnum; import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; +import com.tencent.bk.job.execute.engine.util.TimeoutUtils; import com.tencent.bk.job.execute.model.AccountDTO; import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; @@ -220,6 +221,7 @@ public Long createTaskInstanceFast(TaskInstanceDTO taskInstance, taskInstance, stepInstance); StopWatch watch = new StopWatch("createTaskInstanceFast"); standardizeStepDynamicGroupId(Collections.singletonList(stepInstance)); + adjustStepTimeout(stepInstance); try { // 设置脚本信息 watch.start("checkAndSetScriptInfoIfScriptTask"); @@ -271,7 +273,17 @@ public Long createTaskInstanceFast(TaskInstanceDTO taskInstance, log.warn("CreateTaskInstanceFast is slow, statistics: {}", watch.prettyPrint()); } } + } + /** + * 调整任务超时时间 + * + * @param stepInstance 步骤 + */ + private void adjustStepTimeout(StepInstanceDTO stepInstance) { + if (stepInstance.getTimeout() != null) { + stepInstance.setTimeout(TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout())); + } } private void checkAndSetAccountInfo(StepInstanceDTO stepInstance, @@ -920,6 +932,9 @@ public TaskInstanceDTO createTaskInstanceForTask(TaskExecuteParam executeParam) Map finalVariableValueMap = taskInfo.getVariables(); ServiceTaskPlanDTO jobPlan = taskInfo.getJobPlan(); + // 调整超时时间 + stepInstanceList.forEach(this::adjustStepTimeout); + // 检查高危脚本 watch.start("checkDangerousScript"); batchCheckScriptMatchDangerousRule(taskInstance, stepInstanceList); From 913273c66fd3947b9770286aa6cb922f28ead15f Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 15 Dec 2021 10:44:35 +0800 Subject: [PATCH 18/35] =?UTF-8?q?bugfix:=20=E7=94=A8=E6=88=B7=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E6=96=B9=E6=A1=88=E4=B8=AD=E7=9A=84=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E8=AE=BE=E7=BD=AE=E4=B8=BA0=EF=BC=8C?= =?UTF-8?q?=E4=BD=9C=E4=B8=9A=E5=9C=A8=E6=89=A7=E8=A1=8C5min=E5=90=8E?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E5=A4=B1=E8=B4=A5=20#532?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bk/job/execute/service/impl/TaskExecuteServiceImpl.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index fd28c5c58a..747a134693 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -281,9 +281,7 @@ public Long createTaskInstanceFast(TaskInstanceDTO taskInstance, * @param stepInstance 步骤 */ private void adjustStepTimeout(StepInstanceDTO stepInstance) { - if (stepInstance.getTimeout() != null) { - stepInstance.setTimeout(TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout())); - } + stepInstance.setTimeout(TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout())); } private void checkAndSetAccountInfo(StepInstanceDTO stepInstance, From b0c853401a7704c826ba4ebf719d3d9de90d862c Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 15 Dec 2021 08:05:48 +0800 Subject: [PATCH 19/35] =?UTF-8?q?bugfix:=20=E7=94=A8=E6=88=B7=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E6=96=B9=E6=A1=88=E4=B8=AD=E7=9A=84=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E8=AE=BE=E7=BD=AE=E4=B8=BA0=EF=BC=8C?= =?UTF-8?q?=E4=BD=9C=E4=B8=9A=E5=9C=A8=E6=89=A7=E8=A1=8C5min=E5=90=8E?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E5=A4=B1=E8=B4=A5=20#532?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/TaskExecuteServiceImpl.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index e724ed15d7..fd28c5c58a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -59,6 +59,7 @@ import com.tencent.bk.job.execute.constants.UserOperationEnum; import com.tencent.bk.job.execute.engine.TaskExecuteControlMsgSender; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; +import com.tencent.bk.job.execute.engine.util.TimeoutUtils; import com.tencent.bk.job.execute.model.AccountDTO; import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; @@ -220,6 +221,7 @@ public Long createTaskInstanceFast(TaskInstanceDTO taskInstance, taskInstance, stepInstance); StopWatch watch = new StopWatch("createTaskInstanceFast"); standardizeStepDynamicGroupId(Collections.singletonList(stepInstance)); + adjustStepTimeout(stepInstance); try { // 设置脚本信息 watch.start("checkAndSetScriptInfoIfScriptTask"); @@ -271,7 +273,17 @@ public Long createTaskInstanceFast(TaskInstanceDTO taskInstance, log.warn("CreateTaskInstanceFast is slow, statistics: {}", watch.prettyPrint()); } } + } + /** + * 调整任务超时时间 + * + * @param stepInstance 步骤 + */ + private void adjustStepTimeout(StepInstanceDTO stepInstance) { + if (stepInstance.getTimeout() != null) { + stepInstance.setTimeout(TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout())); + } } private void checkAndSetAccountInfo(StepInstanceDTO stepInstance, @@ -920,6 +932,9 @@ public TaskInstanceDTO createTaskInstanceForTask(TaskExecuteParam executeParam) Map finalVariableValueMap = taskInfo.getVariables(); ServiceTaskPlanDTO jobPlan = taskInfo.getJobPlan(); + // 调整超时时间 + stepInstanceList.forEach(this::adjustStepTimeout); + // 检查高危脚本 watch.start("checkDangerousScript"); batchCheckScriptMatchDangerousRule(taskInstance, stepInstanceList); From 347784ac3afb121738635eb27be3abad4bce8216 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 15 Dec 2021 10:44:35 +0800 Subject: [PATCH 20/35] =?UTF-8?q?bugfix:=20=E7=94=A8=E6=88=B7=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E6=96=B9=E6=A1=88=E4=B8=AD=E7=9A=84=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E8=AE=BE=E7=BD=AE=E4=B8=BA0=EF=BC=8C?= =?UTF-8?q?=E4=BD=9C=E4=B8=9A=E5=9C=A8=E6=89=A7=E8=A1=8C5min=E5=90=8E?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E5=A4=B1=E8=B4=A5=20#532?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bk/job/execute/service/impl/TaskExecuteServiceImpl.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index fd28c5c58a..747a134693 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -281,9 +281,7 @@ public Long createTaskInstanceFast(TaskInstanceDTO taskInstance, * @param stepInstance 步骤 */ private void adjustStepTimeout(StepInstanceDTO stepInstance) { - if (stepInstance.getTimeout() != null) { - stepInstance.setTimeout(TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout())); - } + stepInstance.setTimeout(TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout())); } private void checkAndSetAccountInfo(StepInstanceDTO stepInstance, From 7885cacf21fae00b30fdbd1fd9de47d3d733bffe Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 15 Dec 2021 10:55:09 +0800 Subject: [PATCH 21/35] =?UTF-8?q?bugfix:=20=E7=94=A8=E6=88=B7=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E6=96=B9=E6=A1=88=E4=B8=AD=E7=9A=84=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E8=AE=BE=E7=BD=AE=E4=B8=BA0=EF=BC=8C?= =?UTF-8?q?=E4=BD=9C=E4=B8=9A=E5=9C=A8=E6=89=A7=E8=A1=8C5min=E5=90=8E?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E5=A4=B1=E8=B4=A5=20#532?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 忽略PreCI检查产生的文件 --- src/backend/.gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/backend/.gitignore b/src/backend/.gitignore index b265a41ed1..3f6ec1862d 100644 --- a/src/backend/.gitignore +++ b/src/backend/.gitignore @@ -5,4 +5,7 @@ build release -**/bin \ No newline at end of file +**/bin + +.codecc +build.yml \ No newline at end of file From 9303515b8f21d4c5214af4cb6ecd19a95db12eae Mon Sep 17 00:00:00 2001 From: jsonwan Date: Wed, 15 Dec 2021 12:24:57 +0800 Subject: [PATCH 22/35] =?UTF-8?q?fix:=20=E5=88=86=E5=8F=91=E7=AC=AC?= =?UTF-8?q?=E4=B8=89=E6=96=B9=E6=BA=90=E6=96=87=E4=BB=B6=E5=81=B6=E7=8E=B0?= =?UTF-8?q?=E5=B0=8F=E6=96=87=E4=BB=B6MD5=E4=B8=8D=E6=AD=A3=E7=A1=AE=20#52?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bk/job/common/artifactory/sdk/ArtifactoryClient.java | 7 ++++++- .../bk/job/file/worker/cos/service/DownloadFileTask.java | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java index 8cc29796df..a6164dab2c 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java @@ -409,7 +409,12 @@ public Pair getFileInputStream(String projectId, String repoN } else { log.debug("Content-Length from header is null or blank"); } - return Pair.of(resp.getEntity().getContent(), contentLength); + if (resp.getStatusLine() != null && resp.getStatusLine().getStatusCode() == 200) { + return Pair.of(resp.getEntity().getContent(), contentLength); + } else { + log.info("resp.statusLine={},resp.entity={}", resp.getStatusLine(), resp.getEntity()); + throw new InternalException(ErrorCode.FAIL_TO_REQUEST_THIRD_FILE_SOURCE_DOWNLOAD_GENERIC_FILE); + } } catch (IOException e) { log.error("Fail to getFileInputStream", e); throw new InternalException(ErrorCode.FAIL_TO_REQUEST_THIRD_FILE_SOURCE_DOWNLOAD_GENERIC_FILE); diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java index b5ec6b9ba7..b04cef08f4 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java @@ -104,8 +104,8 @@ public void downloadFileToLocal( fileSizeWrapper.set(fileSize); Pair pair = remoteClient.getFileInputStream(filePath); ins = pair.getLeft(); - long length = pair.getRight(); - if (length != fileSize) { + Long length = pair.getRight(); + if (length != null && length != fileSize) { log.warn("{},fileSize={},ins length={}", filePath, fileSize, length); } currentMd5 = FileUtil.writeInsToFile(ins, targetPath, fileSize, speed, process); From 8a568b54a4daf18583a449e07d0a5196d38cce0a Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Wed, 15 Dec 2021 16:26:49 +0800 Subject: [PATCH 23/35] =?UTF-8?q?bugfix:=20=E8=BF=81=E7=A7=BB=E6=A8=A1?= =?UTF-8?q?=E6=9D=BF=EF=BC=8C=E6=A8=A1=E6=9D=BF=E5=90=8D=E7=A7=B0=E5=86=B2?= =?UTF-8?q?=E7=AA=81=EF=BC=8CAPI=20=E8=BF=94=E5=9B=9E=E7=9A=84http?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E7=A0=81=E4=B8=BA404=EF=BC=8C=E5=B9=B6?= =?UTF-8?q?=E4=B8=94=E6=B2=A1=E6=9C=89=E9=94=99=E8=AF=AF=E4=BF=A1=E6=81=AF?= =?UTF-8?q?=20#536?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../i18n/exception/message.properties | 70 +++++++++++-------- .../i18n/exception/message_en.properties | 69 +++++++++--------- .../i18n/exception/message_en_US.properties | 69 +++++++++--------- .../i18n/exception/message_zh.properties | 70 +++++++++++-------- .../i18n/exception/message_zh_CN.properties | 70 +++++++++++-------- .../impl/TaskTemplateServiceImpl.java | 2 +- 6 files changed, 192 insertions(+), 158 deletions(-) diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties index 2f8bc9c85a..c9c3495b4f 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties @@ -104,6 +104,35 @@ 1243013=默认通知策略未配置，请联系管理员 1243014=账号被引用，禁止删除 1243015=文件大小超过上限{0} +1243016=创建 Tag 失败 +1243017=引用的脚本未上线或被删除 +1243018=新建模版失败 +1243019=更新模版失败 +1243020=保存模版发生未知错误 +1243021=标签已存在 +1243022=错误的变量类型 +1243023=新建执行方案失败 +1243024=更新执行方案失败 +1243025=保存执行方案发生未知错误 +1243026=模版不存在 +1243027=创建调试模版失败 +1243028=同步执行方案失败 +1243029=不支持的步骤类型 +1243030=批量操作失败 +1243031=不支持的任务类型 +1243032=新建步骤失败 +1243033=更新步骤失败 +1243034=删除步骤失败 +1243035=更新文件信息失败 +1243036=脚本版本已存在 +1243037=删除失败！存在关联的定时任务 +1243038=删除失败！执行方案存在关联的定时任务 +1243039=脚本版本Id已存在 +1243040=作业模板名称已存在 +1243041=更新模版失败！并发冲突，请不要多次点击保存 +1243045=同步脚本失败 +1243047=脚本版本不存在 +1243048=凭证不存在 ##业务错误-作业执行(job-execute) 1244001=主机为空 @@ -129,10 +158,17 @@ 1244021=文件任务数量超过限制:{0} 1244022=脚本任务目标服务器数量超过限制:{0} 1244023=本地文件{0}在后台不存在 + +## 业务错误-定时任务(job-crontab) +1245006=定时任务执行时间已失效 +1245007=定时任务结束时间或通知时间已失效 +1245005=更新定时任务获取锁失败 + ##业务错误-日志服务(job-logsvr) 1246001=获取作业执行日志失败 1246002=保存作业执行日志失败 1246003=删除作业执行日志失败 + ##业务错误-文件网关服务(job-file-gateway) 1260001=文件源不存在:{0} 1260002=接入点响应异常：ListFileNode，详情：{0} @@ -147,6 +183,7 @@ 1260012=无法根据标识{0}找到文件源 1260013=文件源服务异常 1260014=通过[{0}]找不到file-worker + ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} 1262002=第三方文件源响应异常：ListObjects，详情：{0} @@ -157,37 +194,8 @@ ##业务错误-用户服务、登录服务 1247001=用户不存在或者未登录 -1243016=创建 Tag 失败 -1243017=引用的脚本未上线或被删除 -1243018=新建模版失败 -1243019=更新模版失败 -1243020=保存模版发生未知错误 -1243021=标签已存在 -1243022=错误的变量类型 -1243023=新建执行方案失败 -1243024=更新执行方案失败 -1243025=保存执行方案发生未知错误 -1243026=模版不存在 -1243027=创建调试模版失败 -1243028=同步执行方案失败 -1243029=不支持的步骤类型 -1243030=批量操作失败 -1243031=不支持的任务类型 -1243032=新建步骤失败 -1243033=更新步骤失败 -1243034=删除步骤失败 -1243035=更新文件信息失败 -1243036=脚本版本已存在 -1243037=删除失败！存在关联的定时任务 -1243038=删除失败！执行方案存在关联的定时任务 -1243039=脚本版本Id已存在 -1243045=同步脚本失败 -1243047=脚本版本不存在 -1243048=凭证不存在 -1245006=定时任务执行时间已失效 -1245007=定时任务结束时间或通知时间已失效 -1245005=更新定时任务获取锁失败 -1243041=更新模版失败！并发冲突，请不要多次点击保存。 + + ##业务错误-备份服务 1249001=从制品库获取节点信息失败 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties index 7ebbf7d502..7d76480be3 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties @@ -104,7 +104,35 @@ 1243013=Default notify policies not configed, please contact system administrator 1243014=Account is referenced and cannot be deleted 1243015=File size exceeds maximum {0} - +1243016=Create tag failed +1243017=Script version not online or deleted +1243018=Create job template failed +1243019=Update job template failed +1243020=Unknown error while saving job template +1243021=Tag already exist +1243022=Wrong variable type +1243023=Create job plan failed +1243024=Update job plan failed +1243025=Unknown error while saving job plan +1243026=Job template not exist +1243027=Error while creating debug plan +1243028=Sync job plan failed +1243029=Unsupported step type +1243030=Batch insert failed +1243031=Unsupported task type +1243032=Create step failed +1243033=Update step failed +1243034=Delete step failed +1243035=Update file info failed +1243036=Script version already exist +1243037=Delete failed! Plan is using by cron job(s) +1243038=Delete failed! Some plan(s) is(are) using by cron job(s) +1243039=Script version id already exist +1243040=Job template name already exist +1243041=Update template acquire lock failed! Please do not click save multiple times! +1243045=Sync script failed +1243047=Script version does not exist +1243048=Credential not exist ## Business error - job-execute 1244001=Host is empty @@ -130,10 +158,17 @@ 1244021=The number of file tasks exceeds the limit:{0} 1244022=The number of script task target servers exceeds the limit: {0} 1244023=The local file {0} not exists in backend + +## Business error - job-crontab +1245006=Cron job execution time already passed +1245007=Cron job end time or notify time already passed +1245005=Acquire lock failed while update cron job + ## Business error - job-logsvr 1246001=Failed to get execution log 1246002=Failed to save execution log 1246003=Failed to delete execution log + ## Business error (job-file-gateway) 1260001=FileSource does not exist:{0} 1260002=FileWorker response error:ListFileNode, Detail:{0} @@ -148,6 +183,7 @@ 1260012=Cannot find file source by code:{0} 1260013=File source service invalid 1260014=Cannot find file-worker by {0} + ## Business error (job-file-worker) 1262001=Third file source response error:ListBucket, Detail:{0} 1262002=Third file source response error:ListObjects, Detail:{0} @@ -158,37 +194,6 @@ ## Business error - User/Login 1247001=User does not exist or is not logged in -1243016=Create tag failed -1243017=Script version not online or deleted -1243018=Create job template failed -1243019=Update job template failed -1243020=Unknown error while saving job template -1243021=Tag already exist -1243022=Wrong variable type -1243023=Create job plan failed -1243024=Update job plan failed -1243025=Unknown error while saving job plan -1243026=Job template not exist -1243027=Error while creating debug plan -1243028=Sync job plan failed -1243029=Unsupported step type -1243030=Batch insert failed -1243031=Unsupported task type -1243032=Create step failed -1243033=Update step failed -1243034=Delete step failed -1243035=Update file info failed -1243036=Script version already exist -1243037=Delete failed! Plan is using by cron job(s) -1243038=Delete failed! Some plan(s) is(are) using by cron job(s) -1243039=Script version id already exist -1243045=Sync script failed -1243047=Script version does not exist -1243048=Credential not exist -1245006=Cron job execution time already passed -1245007=Cron job end time or notify time already passed -1245005=Acquire lock failed while update cron job -1243041=Update template acquire lock failed! Please do not click save multiple times! ## Business error - Backup 1249001=Fail to get node info from artifactory diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties index 7ebbf7d502..7d76480be3 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties @@ -104,7 +104,35 @@ 1243013=Default notify policies not configed, please contact system administrator 1243014=Account is referenced and cannot be deleted 1243015=File size exceeds maximum {0} - +1243016=Create tag failed +1243017=Script version not online or deleted +1243018=Create job template failed +1243019=Update job template failed +1243020=Unknown error while saving job template +1243021=Tag already exist +1243022=Wrong variable type +1243023=Create job plan failed +1243024=Update job plan failed +1243025=Unknown error while saving job plan +1243026=Job template not exist +1243027=Error while creating debug plan +1243028=Sync job plan failed +1243029=Unsupported step type +1243030=Batch insert failed +1243031=Unsupported task type +1243032=Create step failed +1243033=Update step failed +1243034=Delete step failed +1243035=Update file info failed +1243036=Script version already exist +1243037=Delete failed! Plan is using by cron job(s) +1243038=Delete failed! Some plan(s) is(are) using by cron job(s) +1243039=Script version id already exist +1243040=Job template name already exist +1243041=Update template acquire lock failed! Please do not click save multiple times! +1243045=Sync script failed +1243047=Script version does not exist +1243048=Credential not exist ## Business error - job-execute 1244001=Host is empty @@ -130,10 +158,17 @@ 1244021=The number of file tasks exceeds the limit:{0} 1244022=The number of script task target servers exceeds the limit: {0} 1244023=The local file {0} not exists in backend + +## Business error - job-crontab +1245006=Cron job execution time already passed +1245007=Cron job end time or notify time already passed +1245005=Acquire lock failed while update cron job + ## Business error - job-logsvr 1246001=Failed to get execution log 1246002=Failed to save execution log 1246003=Failed to delete execution log + ## Business error (job-file-gateway) 1260001=FileSource does not exist:{0} 1260002=FileWorker response error:ListFileNode, Detail:{0} @@ -148,6 +183,7 @@ 1260012=Cannot find file source by code:{0} 1260013=File source service invalid 1260014=Cannot find file-worker by {0} + ## Business error (job-file-worker) 1262001=Third file source response error:ListBucket, Detail:{0} 1262002=Third file source response error:ListObjects, Detail:{0} @@ -158,37 +194,6 @@ ## Business error - User/Login 1247001=User does not exist or is not logged in -1243016=Create tag failed -1243017=Script version not online or deleted -1243018=Create job template failed -1243019=Update job template failed -1243020=Unknown error while saving job template -1243021=Tag already exist -1243022=Wrong variable type -1243023=Create job plan failed -1243024=Update job plan failed -1243025=Unknown error while saving job plan -1243026=Job template not exist -1243027=Error while creating debug plan -1243028=Sync job plan failed -1243029=Unsupported step type -1243030=Batch insert failed -1243031=Unsupported task type -1243032=Create step failed -1243033=Update step failed -1243034=Delete step failed -1243035=Update file info failed -1243036=Script version already exist -1243037=Delete failed! Plan is using by cron job(s) -1243038=Delete failed! Some plan(s) is(are) using by cron job(s) -1243039=Script version id already exist -1243045=Sync script failed -1243047=Script version does not exist -1243048=Credential not exist -1245006=Cron job execution time already passed -1245007=Cron job end time or notify time already passed -1245005=Acquire lock failed while update cron job -1243041=Update template acquire lock failed! Please do not click save multiple times! ## Business error - Backup 1249001=Fail to get node info from artifactory diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties index 2f8bc9c85a..c9c3495b4f 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties @@ -104,6 +104,35 @@ 1243013=默认通知策略未配置，请联系管理员 1243014=账号被引用，禁止删除 1243015=文件大小超过上限{0} +1243016=创建 Tag 失败 +1243017=引用的脚本未上线或被删除 +1243018=新建模版失败 +1243019=更新模版失败 +1243020=保存模版发生未知错误 +1243021=标签已存在 +1243022=错误的变量类型 +1243023=新建执行方案失败 +1243024=更新执行方案失败 +1243025=保存执行方案发生未知错误 +1243026=模版不存在 +1243027=创建调试模版失败 +1243028=同步执行方案失败 +1243029=不支持的步骤类型 +1243030=批量操作失败 +1243031=不支持的任务类型 +1243032=新建步骤失败 +1243033=更新步骤失败 +1243034=删除步骤失败 +1243035=更新文件信息失败 +1243036=脚本版本已存在 +1243037=删除失败！存在关联的定时任务 +1243038=删除失败！执行方案存在关联的定时任务 +1243039=脚本版本Id已存在 +1243040=作业模板名称已存在 +1243041=更新模版失败！并发冲突，请不要多次点击保存 +1243045=同步脚本失败 +1243047=脚本版本不存在 +1243048=凭证不存在 ##业务错误-作业执行(job-execute) 1244001=主机为空 @@ -129,10 +158,17 @@ 1244021=文件任务数量超过限制:{0} 1244022=脚本任务目标服务器数量超过限制:{0} 1244023=本地文件{0}在后台不存在 + +## 业务错误-定时任务(job-crontab) +1245006=定时任务执行时间已失效 +1245007=定时任务结束时间或通知时间已失效 +1245005=更新定时任务获取锁失败 + ##业务错误-日志服务(job-logsvr) 1246001=获取作业执行日志失败 1246002=保存作业执行日志失败 1246003=删除作业执行日志失败 + ##业务错误-文件网关服务(job-file-gateway) 1260001=文件源不存在:{0} 1260002=接入点响应异常：ListFileNode，详情：{0} @@ -147,6 +183,7 @@ 1260012=无法根据标识{0}找到文件源 1260013=文件源服务异常 1260014=通过[{0}]找不到file-worker + ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} 1262002=第三方文件源响应异常：ListObjects，详情：{0} @@ -157,37 +194,8 @@ ##业务错误-用户服务、登录服务 1247001=用户不存在或者未登录 -1243016=创建 Tag 失败 -1243017=引用的脚本未上线或被删除 -1243018=新建模版失败 -1243019=更新模版失败 -1243020=保存模版发生未知错误 -1243021=标签已存在 -1243022=错误的变量类型 -1243023=新建执行方案失败 -1243024=更新执行方案失败 -1243025=保存执行方案发生未知错误 -1243026=模版不存在 -1243027=创建调试模版失败 -1243028=同步执行方案失败 -1243029=不支持的步骤类型 -1243030=批量操作失败 -1243031=不支持的任务类型 -1243032=新建步骤失败 -1243033=更新步骤失败 -1243034=删除步骤失败 -1243035=更新文件信息失败 -1243036=脚本版本已存在 -1243037=删除失败！存在关联的定时任务 -1243038=删除失败！执行方案存在关联的定时任务 -1243039=脚本版本Id已存在 -1243045=同步脚本失败 -1243047=脚本版本不存在 -1243048=凭证不存在 -1245006=定时任务执行时间已失效 -1245007=定时任务结束时间或通知时间已失效 -1245005=更新定时任务获取锁失败 -1243041=更新模版失败！并发冲突，请不要多次点击保存。 + + ##业务错误-备份服务 1249001=从制品库获取节点信息失败 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties index 2f8bc9c85a..c9c3495b4f 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties @@ -104,6 +104,35 @@ 1243013=默认通知策略未配置，请联系管理员 1243014=账号被引用，禁止删除 1243015=文件大小超过上限{0} +1243016=创建 Tag 失败 +1243017=引用的脚本未上线或被删除 +1243018=新建模版失败 +1243019=更新模版失败 +1243020=保存模版发生未知错误 +1243021=标签已存在 +1243022=错误的变量类型 +1243023=新建执行方案失败 +1243024=更新执行方案失败 +1243025=保存执行方案发生未知错误 +1243026=模版不存在 +1243027=创建调试模版失败 +1243028=同步执行方案失败 +1243029=不支持的步骤类型 +1243030=批量操作失败 +1243031=不支持的任务类型 +1243032=新建步骤失败 +1243033=更新步骤失败 +1243034=删除步骤失败 +1243035=更新文件信息失败 +1243036=脚本版本已存在 +1243037=删除失败！存在关联的定时任务 +1243038=删除失败！执行方案存在关联的定时任务 +1243039=脚本版本Id已存在 +1243040=作业模板名称已存在 +1243041=更新模版失败！并发冲突，请不要多次点击保存 +1243045=同步脚本失败 +1243047=脚本版本不存在 +1243048=凭证不存在 ##业务错误-作业执行(job-execute) 1244001=主机为空 @@ -129,10 +158,17 @@ 1244021=文件任务数量超过限制:{0} 1244022=脚本任务目标服务器数量超过限制:{0} 1244023=本地文件{0}在后台不存在 + +## 业务错误-定时任务(job-crontab) +1245006=定时任务执行时间已失效 +1245007=定时任务结束时间或通知时间已失效 +1245005=更新定时任务获取锁失败 + ##业务错误-日志服务(job-logsvr) 1246001=获取作业执行日志失败 1246002=保存作业执行日志失败 1246003=删除作业执行日志失败 + ##业务错误-文件网关服务(job-file-gateway) 1260001=文件源不存在:{0} 1260002=接入点响应异常：ListFileNode，详情：{0} @@ -147,6 +183,7 @@ 1260012=无法根据标识{0}找到文件源 1260013=文件源服务异常 1260014=通过[{0}]找不到file-worker + ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} 1262002=第三方文件源响应异常：ListObjects，详情：{0} @@ -157,37 +194,8 @@ ##业务错误-用户服务、登录服务 1247001=用户不存在或者未登录 -1243016=创建 Tag 失败 -1243017=引用的脚本未上线或被删除 -1243018=新建模版失败 -1243019=更新模版失败 -1243020=保存模版发生未知错误 -1243021=标签已存在 -1243022=错误的变量类型 -1243023=新建执行方案失败 -1243024=更新执行方案失败 -1243025=保存执行方案发生未知错误 -1243026=模版不存在 -1243027=创建调试模版失败 -1243028=同步执行方案失败 -1243029=不支持的步骤类型 -1243030=批量操作失败 -1243031=不支持的任务类型 -1243032=新建步骤失败 -1243033=更新步骤失败 -1243034=删除步骤失败 -1243035=更新文件信息失败 -1243036=脚本版本已存在 -1243037=删除失败！存在关联的定时任务 -1243038=删除失败！执行方案存在关联的定时任务 -1243039=脚本版本Id已存在 -1243045=同步脚本失败 -1243047=脚本版本不存在 -1243048=凭证不存在 -1245006=定时任务执行时间已失效 -1245007=定时任务结束时间或通知时间已失效 -1245005=更新定时任务获取锁失败 -1243041=更新模版失败！并发冲突，请不要多次点击保存。 + + ##业务错误-备份服务 1249001=从制品库获取节点信息失败 diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateServiceImpl.java index fe92e587d4..a6261f5ab3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateServiceImpl.java @@ -588,7 +588,7 @@ public Long saveTaskTemplateForMigration( TaskTemplateInfoDTO taskTemplateByName = taskTemplateDAO.getTaskTemplateByName(taskTemplateInfo.getAppId(), taskTemplateInfo.getName()); if (taskTemplateByName != null) { - throw new NotFoundException(ErrorCode.TEMPLATE_NAME_EXIST); + throw new AlreadyExistsException(ErrorCode.TEMPLATE_NAME_EXIST); } taskTemplateService.createNewTagForTemplateIfNotExist(taskTemplateInfo); From a66d6d9149fcad1a2f7661dee1ededd330e510f5 Mon Sep 17 00:00:00 2001 From: jsonwan Date: Wed, 15 Dec 2021 19:15:54 +0800 Subject: [PATCH 24/35] =?UTF-8?q?chore:=20=E5=8D=87=E7=BA=A7=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=E5=B7=A5=E5=85=B7shadow=E7=89=88=E6=9C=AC=20#539?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/upgrader/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/upgrader/build.gradle b/src/backend/upgrader/build.gradle index 0d0932d10a..6c6cd6b5e1 100644 --- a/src/backend/upgrader/build.gradle +++ b/src/backend/upgrader/build.gradle @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ plugins { - id "com.github.johnrengelman.shadow" version "5.1.0" + id "com.github.johnrengelman.shadow" version "7.1.1" } ext { if (System.getProperty("jobVersion")) { From 1f2a7fc9b87bf26f44d00bdc8d8d78a0da70df75 Mon Sep 17 00:00:00 2001 From: tyleryuwang Date: Thu, 16 Dec 2021 09:01:52 +0800 Subject: [PATCH 25/35] Update issue templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 规范issue --- .github/ISSUE_TEMPLATE/bug-report.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index bb20b2bc7c..8ba7f2bd82 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -7,4 +7,10 @@ assignees: '' --- +**版本/分支/tag** +**问题描述** + +**如何复现** + +**预期结果** From 5dd40194c4d316e908c3a3fc77b2566bc96494c9 Mon Sep 17 00:00:00 2001 From: hLinx <327159425@qq.com> Date: Mon, 20 Dec 2021 16:33:48 +0800 Subject: [PATCH 26/35] =?UTF-8?q?perf:=20=E8=B6=85=E6=97=B6=E6=97=B6?= =?UTF-8?q?=E9=95=BF=E7=9A=84=E7=BB=9F=E4=B8=80=E5=A4=84=E7=90=86=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20#543?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/strategy/script-timeout.vue | 27 ++++++++++--------- src/frontend/src/i18n/local.js | 1 + 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/frontend/src/components/task-step/script/strategy/script-timeout.vue b/src/frontend/src/components/task-step/script/strategy/script-timeout.vue index 4722b0dd97..15998972d0 100644 --- a/src/frontend/src/components/task-step/script/strategy/script-timeout.vue +++ b/src/frontend/src/components/task-step/script/strategy/script-timeout.vue @@ -27,19 +27,22 @@ diff --git a/src/frontend/src/components/task-step/file/item-factory.vue b/src/frontend/src/components/task-step/file/item-factory.vue index a17717f1c4..84ae4dde53 100644 --- a/src/frontend/src/components/task-step/file/item-factory.vue +++ b/src/frontend/src/components/task-step/file/item-factory.vue @@ -26,11 +26,15 @@ --> diff --git a/src/frontend/src/components/task-step/script/item-factory.vue b/src/frontend/src/components/task-step/script/item-factory.vue index 7769588e6a..b9042fd856 100644 --- a/src/frontend/src/components/task-step/script/item-factory.vue +++ b/src/frontend/src/components/task-step/script/item-factory.vue @@ -26,16 +26,20 @@ -->