diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8512e4569d2..24db0aced65 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: jobs: frontend: name: Build frontend - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: Set up Node.js @@ -34,7 +34,7 @@ jobs: agent: name: Build agent on ${{ matrix.os }} - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/setup-go@v2 with: @@ -50,7 +50,7 @@ jobs: backend: needs: [frontend, agent] name: Build backend and release - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 services: mysql: image: mysql:8.0 @@ -114,7 +114,7 @@ jobs: cloudNative: needs: [frontend, agent] name: Build image and helm chart - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 services: mysql: image: mysql:8.0 @@ -141,7 +141,7 @@ jobs: - name: Gradle Build Backend Service -- CLOUD NATIVE working-directory: src/backend/ci run: | - ./gradlew clean test build -x test :core:worker:worker-agent:shadowJar -Ddevops.assemblyMode=K8S \ + ./gradlew clean test build -x test :core:worker:worker-agent:shadowJar -Ddevops.assemblyMode=KUBERNETES \ -DmysqlURL=127.0.0.1:${{ job.services.mysql.ports['3306'] }} -DmysqlUser=root -DmysqlPasswd=root --no-daemon - name: Get Agent - CLOUD NATIVE uses: actions/download-artifact@v1 @@ -171,7 +171,7 @@ jobs: - name: Setup Python -- CLOUD NATIVE uses: actions/setup-python@v4 with: - python-version: "3.6.8" + python-version: "3.7.15" - name: Generate Helm Chart -- CLOUD NATIVE working-directory: helm-charts/core/ci run: | @@ -196,7 +196,7 @@ jobs: codecc: name: Build CodeCC - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 @@ -236,7 +236,7 @@ jobs: bkrepo: name: Build BkRepo - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 @@ -257,7 +257,7 @@ jobs: releaseAll: name: Release All - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 if: ${{ always() }} needs: [backend, cloudNative, codecc, bkrepo] steps: diff --git a/helm-charts/core/ci/README.md b/helm-charts/core/ci/README.md index 6957abba5a4..25b7e434890 100644 --- a/helm-charts/core/ci/README.md +++ b/helm-charts/core/ci/README.md @@ -27,6 +27,8 @@ $ helm install bkci . 上述命令将使用默认配置在Kubernetes集群中部署bkci, 并输出访问指引。 +部署默认k8s构建机参考[kubernetes-manager部署文档.md](./kubernetes-manager部署文档.md) + ## 卸载Chart 使用以下命令卸载`bkci`: @@ -380,4 +382,3 @@ $ helm install bkci . -f values # 查看默认配置 $ helm show values . ``` - diff --git a/helm-charts/core/ci/build/values.yaml b/helm-charts/core/ci/build/values.yaml index 18360c76caf..9c378683d23 100644 --- a/helm-charts/core/ci/build/values.yaml +++ b/helm-charts/core/ci/build/values.yaml @@ -71,14 +71,6 @@ multiCluster: # 兜底策略, 为空则不用兜底 defaultNamespace: "" -# 构建机资源配置变量 -buildResource: - publicDocker: - enabled: false - k8sBuild: - enabled: true - defaultValue: KUBERNETES - # 内部数据源配置 mysql: image: @@ -472,7 +464,7 @@ dispatch: # dispatchDocker Deployment dispatchDocker: - enabled: false + enabled: true replicas: 1 podLabels: {} resources: @@ -776,7 +768,10 @@ notify: # openapi Deployment openapi: - enabled: true + enabled: false + secret: + enabled: false + content: "" replicas: 1 podLabels: {} resources: diff --git a/helm-charts/core/ci/build_chart.py b/helm-charts/core/ci/build_chart.py index 6b2a259034c..a9c54a88544 100755 --- a/helm-charts/core/ci/build_chart.py +++ b/helm-charts/core/ci/build_chart.py @@ -51,6 +51,14 @@ 'bkCiStreamGitUrl': 'www.github.com', 'bkCiClusterTag': 'devops', 'bkCiRepositoryGithubServer':'repository', + 'bkCiDockerRoutingType':'KUBERNETES', + 'bkCiDockerJobQuotaEnable':'false', + 'bkCiBcsCpu':'8.0', + 'bkCiBcsMemory':'16048', + 'bkCiKubernetesCpu':'8', + 'bkCiKubernetesMemory':'16048', + 'bkCiKubernetesHost': 'http://kubernetes-manager', + 'bkCiKubernetesToken': 'landun' } if os.path.isfile(default_value_json): @@ -79,7 +87,8 @@ '__BK_CI_INFLUXDB_ADDR__': 'http://{{ include "bkci.influxdbHost" . }}:{{ include "bkci.influxdbPort" . }}', '__BK_CI_VERSION__': '{{ .Chart.AppVersion }}', '__BK_CI_DISPATCH_KUBERNETES_NS__': '{{ .Release.Namespace }}', - '__BK_CI_CONSUL_DISCOVERY_TAG__': '{{ .Release.Namespace }}' + '__BK_CI_CONSUL_DISCOVERY_TAG__': '{{ .Release.Namespace }}', + '__BK_CI_PRIVATE_URL__': '{{ if empty .Values.config.bkCiPrivateUrl }}{{ .Release.Name }}-bk-ci-gateway{{ else }}{{ .Values.config.bkCiPrivateUrl }}{{ end }}' } # 读取变量映射 diff --git "a/helm-charts/core/ci/kubernetes-manager\351\203\250\347\275\262\346\226\207\346\241\243.md" "b/helm-charts/core/ci/kubernetes-manager\351\203\250\347\275\262\346\226\207\346\241\243.md" new file mode 100644 index 00000000000..8a57a871684 --- /dev/null +++ "b/helm-charts/core/ci/kubernetes-manager\351\203\250\347\275\262\346\226\207\346\241\243.md" @@ -0,0 +1,41 @@ +# kubernetes-manager +src/backend/dispatch-k8s-manager +## 开发须知 + +1. 修改resource下的config文件时需要同步修改 manifests中的configmap，保持一致。 +2. 修改接口后，需要运行 ./swagger/init-swager.sh 重新初始化swagger文档。 + +## 使用须知 + +kubernetes-manager可以使用二进制方式启动，也可以使用容器方式（更加推荐作为容器启动）。 + +### 以容器方式启动 + +1. 打包镜像。通过修改 makefile 中的 LOCAL_REGISTR与LOCAL_IMAGE，修改默认镜像参数后 make -f ./Makefile image.xxx 打包自己需要的架构。或者直接使用docker文件夹下Dockerfile参考makefile中命令自行打包。打包后即可作为docker容器使用（需配合现有的redis和mysql）。 + +2. 打包chart。通过修改manifests/chart 的Chart.yaml 信息，通过 helm package打包即可。启动时通过阅读并修改values中的内容定制自己需要的启动配置即可（chart包中默认携带mysql以及redis，不需要可以关闭）。 + +3. 补充说明： + - **如何链接不同的kubernetes集群**通过修改 values中的 useKubeConfig 参数即可开启使用指定的kubeconfig，同时修改 chart/template/kubernetes-manager-configmap.yaml 中 kubeConfig.yaml 即可。 + - **登录调试相关** 因为登录调试需要将https链接转为wss与kuberntes通信，所以需要 **指定需要登录调试集群的kubeconfig**，指定方式参考 **如何链接不同的kubernetes集群**。 + - **realResource优化** 优化使用了kubernetes-scheduler-pluign和prometheus的特性，所以需要配置 prometheus同时需要安装 [ci-dispatch-k8s-manager-plugin](https://github.com/TencentBlueKing/ci-dispatch-k8s-manager-plugin) 插件。 + +#### kubernetes-manager和bk-ci同k8s集群同namespace部署 +配置bk-ci helm values +'bkCiKubernetesHost': "http://kubernetes-manager" // 默认kubernetes-manager的service类型为 NodePort +'bkCiKubernetesToken': "landun" // 同kubernetesManager.apiserver.auth.apiToken.value配置 +#### kubernetes-manager和bk-ci同集群不同namespace部署 +配置bk-ci helm values +'bkCiKubernetesHost': "http://kubernetes-manager.{{ .Release.Name }}" // 默认kubernetes-manager的service类型为 NodePort +'bkCiKubernetesToken': "landun" // 同kubernetesManager.apiserver.auth.apiToken.value配置 +#### kubernetes-manager和bk-ci不同集群部署 +配置bk-ci helm values +'bkCiKubernetesHost': "http://node:port" // // 默认kubernetes-manager的service类型为 NodePort +'bkCiKubernetesToken': "landun" // 同kubernetesManager.apiserver.auth.apiToken.value配置 + +### 以二进制的方式启动 + +1. 打包二进制。参考makefile中的 build.xxx 和 release.xxx 同时修改makefile中 CONFIG_DIR，OUT_DIR来存放配置文件和目录文件（配置文件格式可参考 resources 目录）。 + +2. 补充说明： + - 二进制格式启动类似直接镜像启动，可以相互参考。同时二进制格式启动一样不具备mysql和redis，需要自行准备。 diff --git a/helm-charts/core/ci/templates/gateway/deployment.yaml b/helm-charts/core/ci/templates/gateway/deployment.yaml index 203417dfe4d..053f77516f1 100644 --- a/helm-charts/core/ci/templates/gateway/deployment.yaml +++ b/helm-charts/core/ci/templates/gateway/deployment.yaml @@ -81,8 +81,34 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.name + lifecycle: + preStop: + exec: + command: + - sh + - '-c' + - sleep 10s && /usr/local/openresty/nginx/sbin/nginx -s quit + livenessProbe: + exec: + command: + - /bin/sh + - -c + - "[ -f /usr/local/openresty/nginx/run/nginx.pid ] && ps -A | grep nginx" + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + readinessProbe: + httpGet: + path: /nginx_status + port: 80 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + successThreshold: 1 envFrom: - configMapRef: name: {{ include "bkci.names.fullname" . }}-gateway workingDir: /usr/local/openresty/nginx/ + terminationGracePeriodSeconds: 40 {{- end -}} diff --git a/helm-charts/core/ci/templates/init/init.sql.yaml b/helm-charts/core/ci/templates/init/init.sql.yaml index 41ec7ce09b5..938225c4369 100644 --- a/helm-charts/core/ci/templates/init/init.sql.yaml +++ b/helm-charts/core/ci/templates/init/init.sql.yaml @@ -44,6 +44,6 @@ spec: - | for SQL in *.sql; do mysql -u{{- include "bkci.mysqlUsername" . }} -p{{- include "bkci.mysqlPassword" . }} -h{{ $mysqlData._0 }} -P{{ $mysqlData._1 }}< $SQL; done ; echo 'DELETE FROM devops_ci_store.T_CONTAINER WHERE ID = "d51a712508c011e99792fa163e50f2b5";'|mysql -u{{- include "bkci.mysqlUsername" . }} -p{{- include "bkci.mysqlPassword" . }} -h{{ $mysqlData._0 }} -P{{ $mysqlData._1 }} - echo 'update devops_ci_store.T_BUSINESS_CONFIG set CONFIG_VALUE="{{ .Values.buildResource.defaultValue }}" WHERE BUSINESS_VALUE="LINUX" AND FEATURE="defaultBuildType";'|mysql -u{{- include "bkci.mysqlUsername" . }} -p{{- include "bkci.mysqlPassword" . }} -h{{ $mysqlData._0 }} -P{{ $mysqlData._1 }} + echo 'update devops_ci_store.T_BUSINESS_CONFIG set CONFIG_VALUE="DOCKER" WHERE BUSINESS_VALUE="LINUX" AND FEATURE="defaultBuildType";'|mysql -u{{- include "bkci.mysqlUsername" . }} -p{{- include "bkci.mysqlPassword" . }} -h{{ $mysqlData._0 }} -P{{ $mysqlData._1 }} restartPolicy: OnFailure {{- end -}} diff --git a/helm-charts/core/ci/templates/openapi/deployment.yaml b/helm-charts/core/ci/templates/openapi/deployment.yaml index 9d09862b5d9..daaaf581966 100644 --- a/helm-charts/core/ci/templates/openapi/deployment.yaml +++ b/helm-charts/core/ci/templates/openapi/deployment.yaml @@ -113,6 +113,11 @@ spec: - mountPath: /data/workspace/openapi/jvm name: log-volume subPathExpr: bkci/jvm/$(POD_NAME) + {{ if .Values.openapi.secret.enabled }} + - mountPath: {{ .Values.config.bkCiOpenapiApiPubOuter | splitList "/" | initial | join "/" }} + name: bk-key-volume + readOnly: true + {{ end }} lifecycle: preStop: exec: @@ -124,4 +129,9 @@ spec: - hostPath: path: /data name: log-volume + {{ if .Values.openapi.secret.enabled }} + - name: bk-key-volume + secret: + secretName: openapi-bk-key + {{ end }} {{- end -}} diff --git a/helm-charts/core/ci/templates/openapi/secret.yaml b/helm-charts/core/ci/templates/openapi/secret.yaml new file mode 100644 index 00000000000..a9a3b03e5ff --- /dev/null +++ b/helm-charts/core/ci/templates/openapi/secret.yaml @@ -0,0 +1,9 @@ +{{ if and .Values.openapi.enabled .Values.openapi.secret.enabled }} +kind: Secret +apiVersion: v1 +metadata: + name: openapi-bk-key +data: + {{ .Values.config.bkCiOpenapiApiPubOuter | splitList "/" | last }}: {{ .Values.openapi.secret.content }} +type: Opaque +{{ end }} diff --git a/helm-charts/core/ci/templates/store/deployment.yaml b/helm-charts/core/ci/templates/store/deployment.yaml index 3640e057c0f..4a45732c176 100644 --- a/helm-charts/core/ci/templates/store/deployment.yaml +++ b/helm-charts/core/ci/templates/store/deployment.yaml @@ -75,10 +75,6 @@ spec: value: {{ .Chart.Name }} - name: MULTI_CLUSTER value: {{ .Values.multiCluster.enabled | quote }} - - name: ENABLE_PUBLIC_DOCKER - value: {{ .Values.buildResource.publicDocker.enabled | quote }} - - name: ENABLE_K8S_BUILD - value: {{ .Values.buildResource.k8sBuild.enabled | quote }} - name: DEFAULT_NAMESPACE value: {{ .Values.multiCluster.defaultNamespace }} - name: POD_NAME diff --git a/scripts/bkenv.properties b/scripts/bkenv.properties index f74881fd91c..51d9706e74b 100644 --- a/scripts/bkenv.properties +++ b/scripts/bkenv.properties @@ -195,6 +195,8 @@ BK_CI_AGENTLESS_IMAGE_REGISTRY_URL= BK_CI_AGENTLESS_IMAGE_REGISTRY_USER= # BK_CI_DOCKER_ROUTING_TYPE默认为VM. 按需修改. 默认容器调度类型：VM, KUBERNETES, BCS. BK_CI_DOCKER_ROUTING_TYPE=VM +# BK_CI_DOCKER_JOB_QUOTA_ENABLE默认为false. 按需修改. 是否开启job限额. +BK_CI_DOCKER_JOB_QUOTA_ENABLE=false # BK_CI_CODEOA_API_KEY无默认值. 废弃. 待清理配置文件及相关代码. BK_CI_CODEOA_API_KEY= # BK_CI_CODEOA_API_URL无默认值. 废弃. 待清理配置文件及相关代码. @@ -291,6 +293,14 @@ BK_CI_WECHATWORK_TOKEN= BK_CI_WECHATWORK_AESKEY= # 企业微信api配置 api域名 BK_CI_WECHATWORK_URL= +# BK_CI_OPENAPI_API_BLUEKING_ENABLE 用于是否开启blueking api filter +BK_CI_OPENAPI_API_BLUEKING_ENABLE=false +# BK_CI_OPENAPI_API_PUB_OUTER 用于blueking api filter jwt鉴权，内容为pub文件完整路径 +BK_CI_OPENAPI_API_PUB_OUTER= +# BK_CI_OPENAPI_API_AUTH 用于blueking api filter 区分鉴权模式 +BK_CI_OPENAPI_API_AUTH=true +# BK_CI_OPENAPI_VERIFY_PROJECT 在 blueking api filter 中使用，是否开启projectId强校验。 +BK_CI_OPENAPI_VERIFY_PROJECT=false ########## # 4-微服务依赖 @@ -387,6 +397,8 @@ BK_CI_TURBO_REDIS_PASSWORD=$BK_CI_REDIS_PASSWORD BK_CI_TURBO_REDIS_PORT=$BK_CI_REDIS_PORT # BK_REPO_GATEWAY_IP无默认值，按需修改，如部署了蓝鲸制品库bkrepo，并且需要集成到CI，需要配置bkrepo网关的IP地址 BK_REPO_GATEWAY_IP= +# BK_REPO_DOCKER_REGISTRY无默认值，按需修改 +BK_REPO_DOCKER_REGISTRY= # BK_CI_API_TOKEN_ENABLED默认为false BK_CI_API_TOKEN_ENABLED=false # BK_CI_API_TOKEN_SECRET无默认值，按需修改 diff --git a/src/agent/Makefile b/src/agent/Makefile index 4eaefd86e03..bec2b96de7f 100644 --- a/src/agent/Makefile +++ b/src/agent/Makefile @@ -21,21 +21,21 @@ windows: build_windows macos_no_cgo: build_macos_no_cgo build_macos_arm64_no_cgo -build_linux: test +build_linux: test clean mkdir -p $(BINDIR) GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsDaemon_linux $(CMDDIR)/daemon/main.go GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsAgent_linux $(CMDDIR)/agent/main.go GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/upgrader_linux $(CMDDIR)/upgrader/main.go GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/installer_linux $(CMDDIR)/installer/main.go ls -la $(BINDIR) -build_linux_arm64: test +build_linux_arm64: test clean mkdir -p $(BINDIR) GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsDaemon_linux_arm64 $(CMDDIR)/daemon/main.go GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsAgent_linux_arm64 $(CMDDIR)/agent/main.go GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ${BUILD_FLAGS} -o $(BINDIR)/upgrader_linux_arm64 $(CMDDIR)/upgrader/main.go GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ${BUILD_FLAGS} -o $(BINDIR)/installer_linux_arm64 $(CMDDIR)/installer/main.go ls -la $(BINDIR) -build_linux_mips64: test +build_linux_mips64: test clean mkdir -p $(BINDIR) GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=mips64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsDaemon_linux_mips64 $(CMDDIR)/daemon/main.go GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=mips64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsAgent_linux_mips64 $(CMDDIR)/agent/main.go @@ -44,14 +44,14 @@ build_linux_mips64: test ls -la $(BINDIR) # Telegraf 的 cpu和diskio 插件采集使用的 shirou 包需要开启cgo才可以在darwin情况下采集成功 -build_macos: test +build_macos: test clean mkdir -p $(BINDIR) GO111MODULE=on CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsDaemon_macos $(CMDDIR)/daemon/main.go GO111MODULE=on CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsAgent_macos $(CMDDIR)/agent/main.go GO111MODULE=on CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/upgrader_macos $(CMDDIR)/upgrader/main.go GO111MODULE=on CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/installer_macos $(CMDDIR)/installer/main.go ls -la $(BINDIR) -build_macos_arm64: test +build_macos_arm64: test clean mkdir -p $(BINDIR) GO111MODULE=on CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsDaemon_macos_arm64 $(CMDDIR)/daemon/main.go GO111MODULE=on CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsAgent_macos_arm64 $(CMDDIR)/agent/main.go @@ -60,14 +60,14 @@ build_macos_arm64: test ls -la $(BINDIR) # 方便统一交叉编译，这里会提供不开启cgo的macos编译，但是其监控的 cpu和diskio 会没有数据 -build_macos_no_cgo: test +build_macos_no_cgo: test clean mkdir -p $(BINDIR) GO111MODULE=on CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsDaemon_macos $(CMDDIR)/daemon/main.go GO111MODULE=on CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsAgent_macos $(CMDDIR)/agent/main.go GO111MODULE=on CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/upgrader_macos $(CMDDIR)/upgrader/main.go GO111MODULE=on CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build ${BUILD_FLAGS} -o $(BINDIR)/installer_macos $(CMDDIR)/installer/main.go ls -la $(BINDIR) -build_macos_arm64_no_cgo: test +build_macos_arm64_no_cgo: test clean mkdir -p $(BINDIR) GO111MODULE=on CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsDaemon_macos_arm64 $(CMDDIR)/daemon/main.go GO111MODULE=on CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsAgent_macos_arm64 $(CMDDIR)/agent/main.go @@ -75,7 +75,7 @@ build_macos_arm64_no_cgo: test GO111MODULE=on CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build ${BUILD_FLAGS} -o $(BINDIR)/installer_macos_arm64 $(CMDDIR)/installer/main.go ls -la $(BINDIR) -build_windows: test +build_windows: test clean mkdir -p $(BINDIR) GO111MODULE=on CGO_ENABLED=0 GOOS=windows GOARCH=386 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsDaemon.exe $(CMDDIR)/daemon/main_win.go GO111MODULE=on CGO_ENABLED=0 GOOS=windows GOARCH=386 go build ${BUILD_FLAGS} -o $(BINDIR)/devopsAgent.exe $(CMDDIR)/agent/main.go diff --git a/src/agent/go.mod b/src/agent/go.mod index f6bf0170ca7..16372a4f0e0 100644 --- a/src/agent/go.mod +++ b/src/agent/go.mod @@ -3,6 +3,7 @@ module github.com/Tencent/bk-ci/src/agent go 1.17 require ( + github.com/docker/docker v20.10.22+incompatible github.com/gofrs/flock v0.8.1 github.com/influxdata/telegraf v1.22.3 github.com/kardianos/service v1.2.1 @@ -14,6 +15,7 @@ require ( require ( collectd.org v0.5.0 // indirect + github.com/Microsoft/go-winio v0.4.17 // indirect github.com/alecthomas/participle v0.4.1 // indirect github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect github.com/antchfx/jsonquery v1.1.5 // indirect @@ -23,6 +25,9 @@ require ( github.com/caio/go-tdigest v3.1.0+incompatible // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60 // indirect github.com/fatih/color v1.10.0 // indirect github.com/frankban/quicktest v1.14.2 // indirect @@ -43,6 +48,8 @@ require ( github.com/mattn/go-isatty v0.0.12 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect github.com/philhofer/fwd v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect diff --git a/src/agent/go.sum b/src/agent/go.sum index eb525121a78..c36628fd465 100644 --- a/src/agent/go.sum +++ b/src/agent/go.sum @@ -105,6 +105,7 @@ github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3 github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-amqp v0.17.0 h1:HHXa3149nKrI0IZwyM7DRcRy5810t9ZICDutn4BYzj4= github.com/Azure/go-amqp v0.17.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -512,7 +513,6 @@ github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7 github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= -github.com/containerd/containerd v1.5.11 h1:+biZCY9Kns9t2J8L9hOqubjvNQBr1ULdmR7kL+omKoY= github.com/containerd/containerd v1.5.11/go.mod h1:FJl/l1urLXpO3oKDx2No2ouBno2GSI56nTl02HfHeZY= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -639,13 +639,15 @@ github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/ github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.14+incompatible h1:+T9/PRYWNDo5SZl5qS1r9Mo/0Q8AwxKKPtu9S1yxM0w= github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.22+incompatible h1:6jX4yB+NtcbldT90k7vBSaWJDB3i+zkVJT9BEK8kQkk= +github.com/docker/docker v20.10.22+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= @@ -1564,6 +1566,7 @@ github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2J github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -1576,6 +1579,7 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mozilla/tls-observatory v0.0.0-20201209171846-0547674fceff/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= @@ -3030,8 +3034,10 @@ gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/src/agent/src/cmd/agent/main.go b/src/agent/src/cmd/agent/main.go index 8f6831dd91f..44ce067d48f 100644 --- a/src/agent/src/cmd/agent/main.go +++ b/src/agent/src/cmd/agent/main.go @@ -29,10 +29,12 @@ package main import ( "fmt" + "math/rand" "os" "path/filepath" "runtime" "strings" + "time" "github.com/Tencent/bk-ci/src/agent/src/pkg/agent" "github.com/Tencent/bk-ci/src/agent/src/pkg/config" @@ -45,6 +47,14 @@ const ( ) func main() { + rand.Seed(time.Now().UnixNano()) + + defer func() { + if err := recover(); err != nil { + logs.Error("agent main panic: ", err) + } + }() + // 初始化日志 logFilePath := filepath.Join(systemutil.GetWorkDir(), "logs", "devopsAgent.log") err := logs.Init(logFilePath) @@ -76,12 +86,6 @@ func main() { systemutil.ExitProcess(1) } - defer func() { - if err := recover(); err != nil { - logs.Error("panic: ", err) - } - }() - if ok := systemutil.CheckProcess(agentProcess); !ok { logs.Warn("get process lock failed, exit") return diff --git a/src/agent/src/pkg/api/ErrorEnum.go b/src/agent/src/pkg/api/ErrorEnum.go new file mode 100644 index 00000000000..b6beab96cd4 --- /dev/null +++ b/src/agent/src/pkg/api/ErrorEnum.go @@ -0,0 +1,136 @@ +package api + +type ErrorCode int + +const ( + BuildProcessRunError ErrorCode = 2128040 + iota + RecoverRunFileError + LoseRunFileError + MakeTmpDirError + BuildProcessStartError + PrepareScriptCreateError + DockerOsError + DockerRunShInitError + DockerRunShStatError + DockerClientCreateError + DockerImagesFetchError + DockerImagePullError + DockerMakeTmpDirError + DockerMountCreateError + DockerContainerCreateError + DockerContainerStartError + DockerContainerRunError + DockerContainerDoneStatusError + DockerChmodInitshError +) + +type ErrorTypes string + +const ( + System ErrorTypes = "SYSTEM" + User ErrorTypes = "USER" + ThirdParty ErrorTypes = "THIRD_PARTY" + Plugin ErrorTypes = "PLUGIN" +) + +type ErrorEnum struct { + Type ErrorTypes + Code ErrorCode + Message string +} + +var ( + NoErrorEnum = &ErrorEnum{Type: "", Code: 0, Message: ""} + BuildProcessRunErrorEnum = &ErrorEnum{Type: User, Code: BuildProcessRunError, Message: "构建进程执行错误"} + RecoverRunFileErrorEnum = &ErrorEnum{Type: User, Code: RecoverRunFileError, Message: "恢复执行文件失败错误"} + LoseRunFileErrorEnum = &ErrorEnum{Type: User, Code: LoseRunFileError, Message: "丢失执行文件失败错误"} + MakeTmpDirErrorEnum = &ErrorEnum{Type: User, Code: MakeTmpDirError, Message: "创建临时目录失败"} + BuildProcessStartErrorEnum = &ErrorEnum{Type: User, Code: BuildProcessStartError, Message: "启动构建进程失败"} + PrepareScriptCreateErrorEnum = &ErrorEnum{Type: User, Code: PrepareScriptCreateError, Message: "预构建脚本创建失败"} + DockerOsErrorEnum = &ErrorEnum{Type: User, Code: DockerOsError, Message: "目前仅支持linux系统使用第三方docker构建机"} + DockerRunShInitErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerRunShInitError, + Message: "下载Docker构建机初始化脚本失败", + } + DockerRunShStatErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerRunShStatError, + Message: "获取Docker构建机初始化脚本状态失败", + } + DockerClientCreateErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerClientCreateError, + Message: "获取docker客户端错误", + } + DockerImagesFetchErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerImagesFetchError, + Message: "获取docker镜像列表错误", + } + DockerImagePullErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerImagePullError, + Message: "拉取docker镜像失败", + } + DockerMakeTmpDirErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerMakeTmpDirError, + Message: "创建Docker构建临时目录失败", + } + DockerMountCreateErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerMountCreateError, + Message: "准备Docker挂载目录失败", + } + DockerContainerCreateErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerContainerCreateError, + Message: "创建docker容器失败", + } + DockerContainerStartErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerContainerStartError, + Message: "启动docker容器失败", + } + DockerContainerRunErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerContainerRunError, + Message: "docker容器运行失败", + } + DockerContainerDoneStatusErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerContainerDoneStatusError, + Message: "docker容器运行结束时状态码不为0", + } + DockerChmodInitshErrorEnum = &ErrorEnum{ + Type: User, + Code: DockerChmodInitshError, + Message: "docker校验并修改启动脚本权限失败", + } +) + +func (t *ThirdPartyBuildInfo) ToFinish( + success bool, + message string, + errorEnum *ErrorEnum, +) *ThirdPartyBuildWithStatus { + if success || errorEnum == NoErrorEnum { + return &ThirdPartyBuildWithStatus{ + ThirdPartyBuildInfo: *t, + Success: success, + Message: message, + Error: nil, + } + } + return &ThirdPartyBuildWithStatus{ + ThirdPartyBuildInfo: *t, + Success: success, + Message: message, + Error: &Error{ + ErrorType: errorEnum.Type, + ErrorMessage: errorEnum.Message, + ErrorCode: errorEnum.Code, + }, + } +} diff --git a/src/agent/src/pkg/api/api.go b/src/agent/src/pkg/api/api.go index 54a43962bb9..a75ef390999 100644 --- a/src/agent/src/pkg/api/api.go +++ b/src/agent/src/pkg/api/api.go @@ -29,26 +29,34 @@ package api import ( "fmt" - "runtime" - "strconv" - "strings" - "github.com/Tencent/bk-ci/src/agent/src/pkg/config" "github.com/Tencent/bk-ci/src/agent/src/pkg/util/httputil" "github.com/Tencent/bk-ci/src/agent/src/pkg/util/systemutil" + "runtime" + "strconv" ) func buildUrl(url string) string { - if strings.HasPrefix(config.GAgentConfig.Gateway, "http") { - return config.GAgentConfig.Gateway + url - } else { - return "http://" + config.GAgentConfig.Gateway + url - } + return config.GetGateWay() + url } -func Heartbeat(buildInfos []ThirdPartyBuildInfo, jdkVersion []string) (*httputil.DevopsResult, error) { +func Heartbeat( + buildInfos []ThirdPartyBuildInfo, + jdkVersion []string, + dockerTaskList []ThirdPartyDockerTaskInfo, + dockerInitFileMd5 DockerInitFileInfo, +) (*httputil.DevopsResult, error) { url := buildUrl("/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/newHeartbeat") + var taskList []ThirdPartyTaskInfo + for _, info := range buildInfos { + taskList = append(taskList, ThirdPartyTaskInfo{ + ProjectId: info.ProjectId, + BuildId: info.BuildId, + VmSeqId: info.VmSeqId, + Workspace: info.Workspace, + }) + } agentHeartbeatInfo := &AgentHeartbeatInfo{ MasterVersion: config.AgentVersion, SlaveVersion: config.GAgentEnv.SlaveVersion, @@ -57,23 +65,27 @@ func Heartbeat(buildInfos []ThirdPartyBuildInfo, jdkVersion []string) (*httputil ParallelTaskCount: config.GAgentConfig.ParallelTaskCount, AgentInstallPath: systemutil.GetExecutableDir(), StartedUser: systemutil.GetCurrentUser().Username, - TaskList: buildInfos, + TaskList: taskList, Props: AgentPropsInfo{ - Arch: runtime.GOARCH, - JdkVersion: jdkVersion, + Arch: runtime.GOARCH, + JdkVersion: jdkVersion, + DockerInitFileMd5: dockerInitFileMd5, }, + DockerParallelTaskCount: config.GAgentConfig.DockerParallelTaskCount, + DockerTaskList: dockerTaskList, } return httputil.NewHttpClient().Post(url).Body(agentHeartbeatInfo).SetHeaders(config.GAgentConfig.GetAuthHeaderMap()).Execute().IntoDevopsResult() } -func CheckUpgrade(jdkVersion []string) (*httputil.AgentResult, error) { +func CheckUpgrade(jdkVersion []string, dockerInitFileMd5 DockerInitFileInfo) (*httputil.AgentResult, error) { url := buildUrl("/ms/dispatch/api/buildAgent/agent/thirdPartyAgent/upgradeNew") info := &UpgradeInfo{ - WorkerVersion: config.GAgentEnv.SlaveVersion, - GoAgentVersion: config.AgentVersion, - JdkVersion: jdkVersion, + WorkerVersion: config.GAgentEnv.SlaveVersion, + GoAgentVersion: config.AgentVersion, + JdkVersion: jdkVersion, + DockerInitFileInfo: dockerInitFileMd5, } return httputil.NewHttpClient().Post(url).Body(info).SetHeaders(config.GAgentConfig.GetAuthHeaderMap()).Execute().IntoAgentResult() @@ -138,3 +150,29 @@ func DownloadAgentInstallBatchZip(saveFile string) error { config.GAgentConfig.BatchInstallKey)) return httputil.DownloadAgentInstallScript(url, config.GAgentConfig.GetAuthHeaderMap(), saveFile) } + +// AuthHeaderDevopsBuildId log需要的buildId的header +const ( + AuthHeaderDevopsBuildId = "X-DEVOPS-BUILD-ID" + AuthHeaderDevopsVmSeqId = "X-DEVOPS-VM-SID" +) + +func AddLogLine(buildId string, message *LogMessage, vmSeqId string) (*httputil.DevopsResult, error) { + url := buildUrl("/ms/log/api/build/logs") + headers := config.GAgentConfig.GetAuthHeaderMap() + headers[AuthHeaderDevopsBuildId] = buildId + headers[AuthHeaderDevopsVmSeqId] = vmSeqId + return httputil.NewHttpClient(). + Post(url).Body(message).SetHeaders(headers).Execute(). + IntoDevopsResult() +} + +func AddLogRedLine(buildId string, message *LogMessage, vmSeqId string) (*httputil.DevopsResult, error) { + url := buildUrl("/ms/log/api/build/logs/red") + headers := config.GAgentConfig.GetAuthHeaderMap() + headers[AuthHeaderDevopsBuildId] = buildId + headers[AuthHeaderDevopsVmSeqId] = vmSeqId + return httputil.NewHttpClient(). + Post(url).Body(message).SetHeaders(headers).Execute(). + IntoDevopsResult() +} diff --git a/src/agent/src/pkg/api/type.go b/src/agent/src/pkg/api/type.go index a447ef97081..205a72e0b48 100644 --- a/src/agent/src/pkg/api/type.go +++ b/src/agent/src/pkg/api/type.go @@ -36,18 +36,52 @@ type ThirdPartyAgentStartInfo struct { } type ThirdPartyBuildInfo struct { - ProjectId string `json:"projectId"` - BuildId string `json:"buildId"` - VmSeqId string `json:"vmSeqId"` - Workspace string `json:"workspace"` - PipelineId string `json:"pipelineId"` - ToDelTmpFiles []string `json:"-"` // #5806 增加异常时清理脚本文件列表, 不序列化 + ProjectId string `json:"projectId"` + BuildId string `json:"buildId"` + VmSeqId string `json:"vmSeqId"` + Workspace string `json:"workspace"` + PipelineId string `json:"pipelineId"` + ToDelTmpFiles []string `json:"-"` // #5806 增加异常时清理脚本文件列表, 不序列化 + DockerBuildInfo *ThirdPartyDockerBuildInfo `json:"dockerBuildInfo"` + ExecuteCount *int `json:"executeCount"` + ContainerHashId string `json:"containerHashId"` +} + +type ThirdPartyDockerBuildInfo struct { + AgentId string `json:"agentId"` + SecretKey string `json:"secretKey"` + Image string `json:"image"` + Credential *Credential `json:"credential"` + Envs map[string]string `json:"envs"` + DockerResource *DockerResourceOptions +} + +type Credential struct { + User string `json:"user"` + Password string `json:"password"` +} + +type DockerResourceOptions struct { + MemoryLimitBytes int64 + CpuPeriod int64 + CpuQuota int64 + BlkioDeviceWriteBps int64 + BlkioDeviceReadBps int64 + Disk int + Description string } type ThirdPartyBuildWithStatus struct { ThirdPartyBuildInfo Success bool `json:"success"` Message string `json:"message"` + Error *Error `json:"error"` +} + +type Error struct { + ErrorType ErrorTypes `json:"errorType"` + ErrorMessage string `json:"errorMessage"` + ErrorCode ErrorCode `json:"errorCode"` } type PipelineResponse struct { @@ -57,31 +91,48 @@ type PipelineResponse struct { } type AgentHeartbeatInfo struct { - MasterVersion string `json:"masterVersion"` - SlaveVersion string `json:"slaveVersion"` - HostName string `json:"hostName"` - AgentIp string `json:"agentIp"` - ParallelTaskCount int `json:"parallelTaskCount"` - AgentInstallPath string `json:"agentInstallPath"` - StartedUser string `json:"startedUser"` - TaskList []ThirdPartyBuildInfo `json:"taskList"` - Props AgentPropsInfo `json:"props"` + MasterVersion string `json:"masterVersion"` + SlaveVersion string `json:"slaveVersion"` + HostName string `json:"hostName"` + AgentIp string `json:"agentIp"` + ParallelTaskCount int `json:"parallelTaskCount"` + AgentInstallPath string `json:"agentInstallPath"` + StartedUser string `json:"startedUser"` + TaskList []ThirdPartyTaskInfo `json:"taskList"` + Props AgentPropsInfo `json:"props"` + DockerParallelTaskCount int `json:"dockerParallelTaskCount"` + DockerTaskList []ThirdPartyDockerTaskInfo `json:"dockerTaskList"` +} + +type ThirdPartyTaskInfo struct { + ProjectId string `json:"projectId"` + BuildId string `json:"buildId"` + VmSeqId string `json:"vmSeqId"` + Workspace string `json:"workspace"` +} + +type ThirdPartyDockerTaskInfo struct { + ProjectId string `json:"projectId"` + BuildId string `json:"buildId"` + VmSeqId string `json:"vmSeqId"` } type AgentPropsInfo struct { - Arch string `json:"arch"` - JdkVersion []string `json:"jdkVersion"` + Arch string `json:"arch"` + JdkVersion []string `json:"jdkVersion"` + DockerInitFileMd5 DockerInitFileInfo `json:"dockerInitFileMd5"` } type AgentHeartbeatResponse struct { - MasterVersion string `json:"masterVersion"` - SlaveVersion string `json:"slaveVersion"` - AgentStatus string `json:"agentStatus"` - ParallelTaskCount int `json:"parallelTaskCount"` - Envs map[string]string `json:"envs"` - Gateway string `json:"gateway"` - FileGateway string `json:"fileGateway"` - Props AgentPropsResp `json:"props"` + MasterVersion string `json:"masterVersion"` + SlaveVersion string `json:"slaveVersion"` + AgentStatus string `json:"agentStatus"` + ParallelTaskCount int `json:"parallelTaskCount"` + Envs map[string]string `json:"envs"` + Gateway string `json:"gateway"` + FileGateway string `json:"fileGateway"` + Props AgentPropsResp `json:"props"` + DockerParallelTaskCount int `json:"dockerParallelTaskCount"` } type AgentPropsResp struct { @@ -90,15 +141,22 @@ type AgentPropsResp struct { } type UpgradeInfo struct { - WorkerVersion string `json:"workerVersion"` - GoAgentVersion string `json:"goAgentVersion"` - JdkVersion []string `json:"jdkVersion"` + WorkerVersion string `json:"workerVersion"` + GoAgentVersion string `json:"goAgentVersion"` + JdkVersion []string `json:"jdkVersion"` + DockerInitFileInfo DockerInitFileInfo `json:"dockerInitFileInfo"` +} + +type DockerInitFileInfo struct { + FileMd5 string `json:"fileMd5"` + NeedUpgrade bool `json:"needUpgrade"` } type UpgradeItem struct { - Agent bool `json:"agent"` - Worker bool `json:"worker"` - Jdk bool `json:"jdk"` + Agent bool `json:"agent"` + Worker bool `json:"worker"` + Jdk bool `json:"jdk"` + DockerInitFile bool `json:"dockerInitFile"` } func NewPipelineResponse(seqId string, status string, response string) *PipelineResponse { @@ -108,3 +166,22 @@ func NewPipelineResponse(seqId string, status string, response string) *Pipeline Response: response, } } + +type LogType string + +const ( + LogtypeLog LogType = "LOG" + LogtypeDebug LogType = "DEBUG" + LogtypeError LogType = "ERROR" + LogtypeWarn LogType = "WARN" +) + +type LogMessage struct { + Message string `json:"message"` + Timestamp int64 `json:"timestamp"` // Millis + Tag string `json:"tag"` + JobId string `json:"jobId"` + LogType LogType `json:"logType"` + ExecuteCount *int `json:"executeCount"` + SubTag *string `json:"subTag"` +} diff --git a/src/agent/src/pkg/collector/collector.go b/src/agent/src/pkg/collector/collector.go index 4b1a32e4132..8a1a62ba1da 100644 --- a/src/agent/src/pkg/collector/collector.go +++ b/src/agent/src/pkg/collector/collector.go @@ -215,6 +215,12 @@ const configTemplateWindows = `[global_tags] ` func DoAgentCollect() { + defer func() { + if err := recover(); err != nil { + logs.Error("agent collect panic: ", err) + } + }() + if config.GAgentConfig.CollectorOn == false { logs.Info("agent collector off") return diff --git a/src/agent/src/pkg/config/config.go b/src/agent/src/pkg/config/config.go index c693fe3de6d..3f300c8a77a 100644 --- a/src/agent/src/pkg/config/config.go +++ b/src/agent/src/pkg/config/config.go @@ -34,12 +34,13 @@ import ( "errors" "fmt" "gopkg.in/ini.v1" - "io/ioutil" "net/http" "os" "path/filepath" + "regexp" "strconv" "strings" + "sync" "github.com/Tencent/bk-ci/src/agent/src/pkg/logs" "github.com/Tencent/bk-ci/src/agent/src/pkg/util" @@ -63,28 +64,32 @@ const ( KeyIgnoreLocalIps = "devops.agent.ignoreLocalIps" KeyBatchInstall = "devops.agent.batch.install" KeyLogsKeepHours = "devops.agent.logs.keep.hours" - // 这个key不会预先出现在配置文件中，因为workdir未知，需要第一次动态获取 - KeyJdkDirPath = "devops.agent.jdk.dir.path" + // KeyJdkDirPath 这个key不会预先出现在配置文件中，因为workdir未知，需要第一次动态获取 + KeyJdkDirPath = "devops.agent.jdk.dir.path" + KeyDockerTaskCount = "devops.docker.parallel.task.count" + keyEnableDockerBuild = "devops.docker.enable" ) // AgentConfig Agent 配置 type AgentConfig struct { - Gateway string - FileGateway string - BuildType string - ProjectId string - AgentId string - SecretKey string - ParallelTaskCount int - EnvType string - SlaveUser string - CollectorOn bool - TimeoutSec int64 - DetectShell bool - IgnoreLocalIps string - BatchInstallKey string - LogsKeepHours int - JdkDirPath string + Gateway string + FileGateway string + BuildType string + ProjectId string + AgentId string + SecretKey string + ParallelTaskCount int + EnvType string + SlaveUser string + CollectorOn bool + TimeoutSec int64 + DetectShell bool + IgnoreLocalIps string + BatchInstallKey string + LogsKeepHours int + JdkDirPath string + DockerParallelTaskCount int + EnableDockerBuild bool } // AgentEnv Agent 环境配置 @@ -193,6 +198,9 @@ func DetectWorkerVersionByDir(workDir string) string { // parseWorkerVersion 解析worker版本 func parseWorkerVersion(output string) string { + // 用正则匹配正确的版本信息，当正则式出错时(versionRegexp = nil)，继续使用原逻辑 + // 主要解决tmp空间不足的情况下，jvm会打印出提示信息，导致识别不到worker版本号 + versionRegexp := regexp.MustCompile(`^v(\d+\.)(\d+\.)(\d+)((-RELEASE)|(-SNAPSHOT)?)$`) lines := strings.Split(output, "\n") for _, line := range lines { line = strings.TrimSpace(line) @@ -200,6 +208,14 @@ func parseWorkerVersion(output string) string { if len(line) > 64 { line = line[:64] } + if versionRegexp != nil { + if versionRegexp.MatchString(line) { + logs.Info("worker version: ", line) + return line + } else { + continue + } + } logs.Info("worker version: ", line) return line } @@ -288,6 +304,17 @@ func LoadAgentConfig() error { jdkDirPath = getJavaDir() } + // 兼容旧版本 .agent.properties 没有这个键 + dockerParallelTaskCount := 4 + if conf.Section("").HasKey(KeyDockerTaskCount) { + dockerParallelTaskCount, err = conf.Section("").Key(KeyDockerTaskCount).Int() + if err != nil || dockerParallelTaskCount < 0 { + return errors.New("invalid dockerParallelTaskCount") + } + } + + enableDocker := conf.Section("").Key(keyEnableDockerBuild).MustBool(false) + GAgentConfig.LogsKeepHours = logsKeepHours GAgentConfig.BatchInstallKey = strings.TrimSpace(conf.Section("").Key(KeyBatchInstall).String()) @@ -323,12 +350,24 @@ func LoadAgentConfig() error { logs.Info("logsKeepHours: ", GAgentConfig.LogsKeepHours) GAgentConfig.JdkDirPath = jdkDirPath logs.Info("jdkDirPath: ", GAgentConfig.JdkDirPath) + GAgentConfig.DockerParallelTaskCount = dockerParallelTaskCount + logs.Info("DockerParallelTaskCount: ", GAgentConfig.DockerParallelTaskCount) + GAgentConfig.EnableDockerBuild = enableDocker + logs.Info("EnableDockerBuild: ", GAgentConfig.EnableDockerBuild) // 初始化 GAgentConfig 写入一次配置, 往文件中写入一次程序中新添加的 key return GAgentConfig.SaveConfig() } +// 可能存在不同协诚写入文件的操作，加上锁保险些 +var saveConfigLock = sync.Mutex{} + // SaveConfig 将配置回写到agent.properties文件保存 func (a *AgentConfig) SaveConfig() error { + saveConfigLock.Lock() + defer func() { + saveConfigLock.Unlock() + }() + filePath := systemutil.GetWorkDir() + "/.agent.properties" content := bytes.Buffer{} @@ -345,8 +384,10 @@ func (a *AgentConfig) SaveConfig() error { content.WriteString(KeyIgnoreLocalIps + "=" + GAgentConfig.IgnoreLocalIps + "\n") content.WriteString(KeyLogsKeepHours + "=" + strconv.Itoa(GAgentConfig.LogsKeepHours) + "\n") content.WriteString(KeyJdkDirPath + "=" + GAgentConfig.JdkDirPath + "\n") + content.WriteString(KeyDockerTaskCount + "=" + strconv.Itoa(GAgentConfig.DockerParallelTaskCount) + "\n") + content.WriteString(keyEnableDockerBuild + "=" + strconv.FormatBool(GAgentConfig.EnableDockerBuild) + "\n") - err := ioutil.WriteFile(filePath, []byte(content.String()), 0666) + err := os.WriteFile(filePath, []byte(content.String()), 0666) if err != nil { logs.Error("write config failed:", err.Error()) return errors.New("write config failed") @@ -378,7 +419,11 @@ func SaveJdkDir(dir string) { return } GAgentConfig.JdkDirPath = dir - GAgentConfig.SaveConfig() + err := GAgentConfig.SaveConfig() + if err != nil { + logs.Error("config.go|SaveJdkDir(dir=%s) failed: %s", dir, err.Error()) + return + } } // getJavaDir 获取本地java文件夹 @@ -390,6 +435,18 @@ func getJavaDir() string { return workDir + "/jdk" } +func GetDockerInitFilePath() string { + return systemutil.GetWorkDir() + "/" + DockerInitFile +} + +func GetGateWay() string { + if strings.HasPrefix(GAgentConfig.Gateway, "http") || strings.HasPrefix(GAgentConfig.Gateway, "https") { + return GAgentConfig.Gateway + } else { + return "http://" + GAgentConfig.Gateway + } +} + // initCert 初始化证书 func initCert() { AbsCertFilePath := systemutil.GetWorkDir() + "/" + CertFilePath @@ -405,7 +462,7 @@ func initCert() { return } // Load client cert - caCert, err := ioutil.ReadFile(AbsCertFilePath) + caCert, err := os.ReadFile(AbsCertFilePath) if err != nil { logs.Warn("Reading server certificate: %s", err) return diff --git a/src/agent/src/pkg/config/config_test.go b/src/agent/src/pkg/config/config_test.go new file mode 100644 index 00000000000..7f201f5c266 --- /dev/null +++ b/src/agent/src/pkg/config/config_test.go @@ -0,0 +1,71 @@ +package config + +import ( + "github.com/Tencent/bk-ci/src/agent/src/pkg/logs" + "os" + "reflect" + "testing" +) + +func Test_parseWorkerVersion(t *testing.T) { + logFile := "config_unit_test.log" + _ = logs.Init(logFile) + + defer func() { _ = os.Remove(logFile) }() + + tests := []struct { + name string + lines string + want string + }{ + { + name: "Insufficient memory", + lines: "OpenJDK 64-Bit Server VM warning: Insufficient space for shared memory file:\n" + + " 30458\n" + + "Try using the -Djava.io.tmpdir= option to select an alternate temp location.\n\n" + + "v1.13.8-RELEASE", + want: "v1.13.8-RELEASE", + }, + { + name: "Normal: with RELEASE", + lines: "v1.13.8-RELEASE\n", + want: "v1.13.8-RELEASE", + }, + { + name: "Normal: without RELEASE", + lines: "v1.13.8\r\n", + want: "v1.13.8", + }, + { + name: "Normal: with SNAPSHOT", + lines: "v1.13.8-SNAPSHOT\r\n", + want: "v1.13.8-SNAPSHOT", + }, + { + name: "illegal: only number", + lines: "12356\n", + want: "", + }, + { + name: "illegal: bad format", + lines: "v3.1.1.1", + want: "", + }, + { + name: "illegal: end with -", + lines: "v1.13.8-\r\n", + want: "", + }, + } + + for _, tt := range tests { + + t.Run(tt.name, func(t *testing.T) { + version := parseWorkerVersion(tt.lines) + if !reflect.DeepEqual(version, tt.want) { + t.Fatalf("Fail: %v = %v, want %v", tt.name, version, tt.want) + } + }) + } + +} diff --git a/src/agent/src/pkg/config/constant.go b/src/agent/src/pkg/config/constant.go index f69602ffb7b..5a4062eab20 100644 --- a/src/agent/src/pkg/config/constant.go +++ b/src/agent/src/pkg/config/constant.go @@ -68,6 +68,8 @@ const ( WorkAgentFile = "worker-agent.jar" JdkClientFile = "jdk.zip" + + DockerInitFile = "agent_docker_init.sh" ) // Auth Header diff --git a/src/agent/src/pkg/config/version.go b/src/agent/src/pkg/config/version.go index fac4815c7b1..7bca20ff581 100644 --- a/src/agent/src/pkg/config/version.go +++ b/src/agent/src/pkg/config/version.go @@ -27,7 +27,7 @@ package config -const AgentVersion = "v1.9.12" +const AgentVersion = "v1.9.16" var ( GitCommit string diff --git a/src/agent/src/pkg/cron/cron.go b/src/agent/src/pkg/cron/cron.go index 2e33d75c79c..23e2d05b6d7 100644 --- a/src/agent/src/pkg/cron/cron.go +++ b/src/agent/src/pkg/cron/cron.go @@ -30,6 +30,7 @@ package cron import ( "fmt" "github.com/Tencent/bk-ci/src/agent/src/pkg/config" + "github.com/Tencent/bk-ci/src/agent/src/pkg/job" "io/ioutil" "os" "strings" @@ -41,6 +42,12 @@ import ( ) func CleanJob() { + defer func() { + if err := recover(); err != nil { + logs.Error("agent clean panic: ", err) + } + }() + intervalInHours := 2 TryCleanFile() for { @@ -115,4 +122,29 @@ func cleanLogFile(timeBeforeInHours int) { } } logs.Info("clean log file done") + + // 清理docker构建记录 + dockerLogDir := job.LocalDockerWorkSpaceDirName + "/logs" + dockerFiles, err := ioutil.ReadDir(dockerLogDir) + if err != nil { + logs.Warn("read docker log dir error: ", err.Error()) + return + } + + // 因为docker构建机是按照buildId分类存储到文件夹中，所以只需要查看文件夹变更日期之后删除即可 + for _, file := range dockerFiles { + if !file.IsDir() { + continue + } + + if int(time.Since(file.ModTime()).Hours()) > timeBeforeInHours { + dockerFullName := dockerLogDir + "/" + file.Name() + err = os.RemoveAll(dockerFullName) + if err != nil { + logs.Warn(fmt.Sprintf("remove docker log file %s failed: ", dockerFullName)) + } else { + logs.Info(fmt.Sprintf("docker log file %s removed", dockerFullName)) + } + } + } } diff --git a/src/agent/src/pkg/heartbeat/heartbeat.go b/src/agent/src/pkg/heartbeat/heartbeat.go index 3a70cdb945f..a854058a977 100644 --- a/src/agent/src/pkg/heartbeat/heartbeat.go +++ b/src/agent/src/pkg/heartbeat/heartbeat.go @@ -41,6 +41,12 @@ import ( ) func DoAgentHeartbeat() { + defer func() { + if err := recover(); err != nil { + logs.Error("agent heartbeat panic: ", err) + } + }() + for { _ = agentHeartbeat() time.Sleep(10 * time.Second) @@ -49,11 +55,18 @@ func DoAgentHeartbeat() { func agentHeartbeat() error { var jdkVersion []string - version := upgrade.JdkVersion.Version.Load() + version := upgrade.JdkVersion.GetVersion() if version != nil { - jdkVersion = version.([]string) + jdkVersion = version } - result, err := api.Heartbeat(job.GBuildManager.GetInstances(), jdkVersion) + result, err := api.Heartbeat( + job.GBuildManager.GetInstances(), + jdkVersion, + job.GBuildDockerManager.GetInstances(), + api.DockerInitFileInfo{ + FileMd5: upgrade.DockerFileMd5.Md5, + NeedUpgrade: upgrade.DockerFileMd5.NeedUpgrade, + }) if err != nil { logs.Error("agent heartbeat failed: ", err.Error()) return errors.New("agent heartbeat failed") @@ -90,6 +103,10 @@ func agentHeartbeat() error { config.GAgentConfig.FileGateway = heartbeatResponse.FileGateway configChanged = true } + if config.GAgentConfig.DockerParallelTaskCount != heartbeatResponse.DockerParallelTaskCount { + config.GAgentConfig.DockerParallelTaskCount = heartbeatResponse.DockerParallelTaskCount + configChanged = true + } if heartbeatResponse.Props.KeepLogsHours > 0 && config.GAgentConfig.LogsKeepHours != heartbeatResponse.Props.KeepLogsHours { diff --git a/src/agent/src/pkg/job/build.go b/src/agent/src/pkg/job/build.go index db78eedc3d4..29863dff6c2 100644 --- a/src/agent/src/pkg/job/build.go +++ b/src/agent/src/pkg/job/build.go @@ -36,6 +36,7 @@ import ( "io/ioutil" "os" "strings" + "sync" "time" "github.com/Tencent/bk-ci/src/agent/src/pkg/api" @@ -48,6 +49,17 @@ import ( "github.com/Tencent/bk-ci/src/agent/src/pkg/util/systemutil" ) +type BuildTotalManagerType struct { + // Lock 多协程修改时的执行锁，这个锁主要用来判断当前是否还有任务，所以添加了任务就可以解锁了 + Lock sync.Mutex +} + +var BuildTotalManager *BuildTotalManagerType + +func init() { + BuildTotalManager = new(BuildTotalManagerType) +} + const buildIntervalInSeconds = 5 // AgentStartup 上报构建机启动 @@ -95,33 +107,50 @@ func DoPollAndBuild() { continue } - instanceCount := GBuildManager.GetInstanceCount() - if config.GAgentConfig.ParallelTaskCount != 0 && instanceCount >= config.GAgentConfig.ParallelTaskCount { - logs.Info(fmt.Sprintf("parallel task count exceed , wait job done, "+ - "ParallelTaskCount config: %d, instance count: %d", - config.GAgentConfig.ParallelTaskCount, instanceCount)) + dockerCanRun, normalCanRun := checkParallelTaskCount() + if !dockerCanRun && !normalCanRun { continue } - // 在接取任务先获取锁，防止与其他任务产生干扰 - GBuildManager.Lock.Lock() + // 在接取任务先获取锁，防止与其他操作产生干扰 + BuildTotalManager.Lock.Lock() buildInfo, err := getBuild() if err != nil { logs.Error("get build failed, retry, err", err.Error()) - GBuildManager.Lock.Unlock() + BuildTotalManager.Lock.Unlock() continue } if buildInfo == nil { logs.Info("no build to run, skip") - GBuildManager.Lock.Unlock() + BuildTotalManager.Lock.Unlock() + continue + } + + logs.Info("build info ", buildInfo, " dockerCanRun ", dockerCanRun) + + if buildInfo.DockerBuildInfo != nil && dockerCanRun { + // 接取job任务之后才可以解除总任务锁解锁 + GBuildDockerManager.AddBuild(buildInfo.BuildId, &api.ThirdPartyDockerTaskInfo{ + ProjectId: buildInfo.ProjectId, + BuildId: buildInfo.BuildId, + VmSeqId: buildInfo.VmSeqId, + }) + BuildTotalManager.Lock.Unlock() + + runDockerBuild(buildInfo) + continue + } + + if !normalCanRun { + BuildTotalManager.Lock.Unlock() continue } // 接取任务之后解锁 GBuildManager.AddPreInstance(buildInfo.BuildId) - GBuildManager.Lock.Unlock() + BuildTotalManager.Lock.Unlock() err = runBuild(buildInfo) if err != nil { @@ -130,6 +159,33 @@ func DoPollAndBuild() { } } +// checkParallelTaskCount 检查当前运行的最大任务数 +func checkParallelTaskCount() (dockerCanRun bool, normalCanRun bool) { + // 检查docker任务 + dockerInstanceCount := GBuildDockerManager.GetInstanceCount() + if config.GAgentConfig.DockerParallelTaskCount != 0 && dockerInstanceCount >= config.GAgentConfig.DockerParallelTaskCount { + logs.Info(fmt.Sprintf("DOCKER_JOB|parallel docker task count exceed , wait job done, "+ + "maxJob config: %d, instance count: %d", + config.GAgentConfig.DockerParallelTaskCount, dockerInstanceCount)) + dockerCanRun = false + } else { + dockerCanRun = true + } + + // 检查普通任务 + instanceCount := GBuildManager.GetInstanceCount() + if config.GAgentConfig.ParallelTaskCount != 0 && instanceCount >= config.GAgentConfig.ParallelTaskCount { + logs.Info(fmt.Sprintf("parallel task count exceed , wait job done, "+ + "ParallelTaskCount config: %d, instance count: %d", + config.GAgentConfig.ParallelTaskCount, instanceCount)) + normalCanRun = false + } else { + normalCanRun = true + } + + return dockerCanRun, normalCanRun +} + // getBuild 从服务器认领要构建的信息 func getBuild() (*api.ThirdPartyBuildInfo, error) { logs.Info("get build") @@ -176,7 +232,7 @@ func runBuild(buildInfo *api.ThirdPartyBuildInfo) error { "\nRestore %s failed, `run install.sh` or `unzip agent.zip` in %s.", agentJarPath, workDir, agentJarPath, workDir) logs.Error(errorMsg) - workerBuildFinish(&api.ThirdPartyBuildWithStatus{ThirdPartyBuildInfo: *buildInfo, Message: errorMsg}) + workerBuildFinish(buildInfo.ToFinish(false, errorMsg, api.RecoverRunFileErrorEnum)) } else { // #5806 替换后修正版本号 if config.GAgentEnv.SlaveVersion != upgradeWorkerFileVersion { config.GAgentEnv.SlaveVersion = upgradeWorkerFileVersion @@ -188,7 +244,7 @@ func runBuild(buildInfo *api.ThirdPartyBuildInfo) error { "\nMissing %s, `run install.sh` or `unzip agent.zip` in %s.", agentJarPath, workDir, agentJarPath, workDir) logs.Error(errorMsg) - workerBuildFinish(&api.ThirdPartyBuildWithStatus{ThirdPartyBuildInfo: *buildInfo, Message: errorMsg}) + workerBuildFinish(buildInfo.ToFinish(false, errorMsg, api.LoseRunFileErrorEnum)) } } @@ -216,7 +272,7 @@ func runBuild(buildInfo *api.ThirdPartyBuildInfo) error { if tmpMkErr != nil { errMsg := fmt.Sprintf("创建临时目录失败(create tmp directory failed): %s", tmpMkErr.Error()) logs.Error(errMsg) - workerBuildFinish(&api.ThirdPartyBuildWithStatus{ThirdPartyBuildInfo: *buildInfo, Message: errMsg}) + workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.MakeTmpDirErrorEnum)) return tmpMkErr } if systemutil.IsWindows() { @@ -236,7 +292,7 @@ func runBuild(buildInfo *api.ThirdPartyBuildInfo) error { if err != nil { errMsg := "start worker process failed: " + err.Error() logs.Error(errMsg) - workerBuildFinish(&api.ThirdPartyBuildWithStatus{ThirdPartyBuildInfo: *buildInfo, Message: errMsg}) + workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.BuildProcessStartErrorEnum)) return err } // 添加需要构建结束后删除的文件 @@ -250,14 +306,14 @@ func runBuild(buildInfo *api.ThirdPartyBuildInfo) error { if err != nil { errMsg := "准备构建脚本生成失败(create start script failed): " + err.Error() logs.Error(errMsg) - workerBuildFinish(&api.ThirdPartyBuildWithStatus{ThirdPartyBuildInfo: *buildInfo, Message: errMsg}) + workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.PrepareScriptCreateErrorEnum)) return err } pid, err := command.StartProcess(startScriptFile, []string{}, workDir, goEnv, runUser) if err != nil { errMsg := "启动构建进程失败(start worker process failed): " + err.Error() logs.Error(errMsg) - workerBuildFinish(&api.ThirdPartyBuildWithStatus{ThirdPartyBuildInfo: *buildInfo, Message: errMsg}) + workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.BuildProcessStartErrorEnum)) return err } GBuildManager.AddBuild(pid, buildInfo) diff --git a/src/agent/src/pkg/job/build_docker.go b/src/agent/src/pkg/job/build_docker.go new file mode 100644 index 00000000000..3c545e60c64 --- /dev/null +++ b/src/agent/src/pkg/job/build_docker.go @@ -0,0 +1,458 @@ +package job + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/Tencent/bk-ci/src/agent/src/pkg/api" + "github.com/Tencent/bk-ci/src/agent/src/pkg/config" + "github.com/Tencent/bk-ci/src/agent/src/pkg/logs" + "github.com/Tencent/bk-ci/src/agent/src/pkg/upgrade/download" + "github.com/Tencent/bk-ci/src/agent/src/pkg/util" + "github.com/Tencent/bk-ci/src/agent/src/pkg/util/systemutil" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/client" + "github.com/pkg/errors" +) + +// buildDockerManager docker构建机构建对象管理 +type buildDockerManager struct { + // instances 正在执行中的构建对象 [string]*api.ThirdPartyDockerTaskInfo + instances sync.Map +} + +func (b *buildDockerManager) GetInstanceCount() int { + var i = 0 + b.instances.Range(func(key, value interface{}) bool { + i++ + return true + }) + return i +} + +func (b *buildDockerManager) GetInstances() []api.ThirdPartyDockerTaskInfo { + result := make([]api.ThirdPartyDockerTaskInfo, 0) + b.instances.Range(func(key, value interface{}) bool { + result = append(result, *value.(*api.ThirdPartyDockerTaskInfo)) + return true + }) + return result +} + +func (b *buildDockerManager) AddBuild(buildId string, info *api.ThirdPartyDockerTaskInfo) { + b.instances.Store(buildId, info) +} + +func (b *buildDockerManager) RemoveBuild(buildId string) { + b.instances.Delete(buildId) +} + +var GBuildDockerManager *buildDockerManager + +func init() { + GBuildDockerManager = new(buildDockerManager) +} + +const ( + entryPointCmd = "/data/init.sh" + localDockerBuildTmpDirName = "docker_build_tmp" + LocalDockerWorkSpaceDirName = "docker_workspace" + dockerDataDir = "/data/landun/workspace" + dockerLogDir = "/data/logs" +) + +func runDockerBuild(buildInfo *api.ThirdPartyBuildInfo) { + if !systemutil.IsLinux() { + GBuildDockerManager.RemoveBuild(buildInfo.BuildId) + dockerBuildFinish(buildInfo.ToFinish(false, "目前仅支持linux系统使用docker构建机", api.DockerOsErrorEnum)) + return + } + // 第一次使用docker构建机后，则直接开启docker构建机相关逻辑 + if !config.GAgentConfig.EnableDockerBuild { + config.GAgentConfig.EnableDockerBuild = true + go config.GAgentConfig.SaveConfig() + } + + // 兼容旧数据，对于没有docker文件的需要重新下载，防止重复下载所以放到主流程做 + if _, err := os.Stat(config.GetDockerInitFilePath()); err != nil { + if os.IsNotExist(err) { + _, err = download.DownloadDockerInitFile(systemutil.GetWorkDir()) + if err != nil { + GBuildDockerManager.RemoveBuild(buildInfo.BuildId) + dockerBuildFinish(buildInfo.ToFinish(false, "下载Docker构建机初始化脚本失败|"+err.Error(), api.DockerRunShInitErrorEnum)) + return + } + } else { + GBuildDockerManager.RemoveBuild(buildInfo.BuildId) + dockerBuildFinish(buildInfo.ToFinish(false, "获取Docker构建机初始化脚本状态失败|"+err.Error(), api.DockerRunShStatErrorEnum)) + return + } + } + + // 每次执行前都校验并修改一次dockerfile权限，防止用户修改或者升级丢失权限 + if err := systemutil.Chmod(config.GetDockerInitFilePath(), os.ModePerm); err != nil { + GBuildDockerManager.RemoveBuild(buildInfo.BuildId) + dockerBuildFinish(buildInfo.ToFinish(false, "校验并修改Docker启动脚本权限失败|"+err.Error(), api.DockerChmodInitshErrorEnum)) + return + } + + go doDockerJob(buildInfo) +} + +const longLogTag = "toolong" + +// doDockerJob 使用docker启动构建 +func doDockerJob(buildInfo *api.ThirdPartyBuildInfo) { + // 各种情况退出时减运行任务数量 + defer func() { + GBuildDockerManager.RemoveBuild(buildInfo.BuildId) + }() + + workDir := systemutil.GetWorkDir() + + dockerBuildInfo := buildInfo.DockerBuildInfo + ctx := context.Background() + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + logs.Error("DOCKER_JOB|create docker client error ", err) + dockerBuildFinish(buildInfo.ToFinish(false, "获取docker客户端错误|"+err.Error(), api.DockerClientCreateErrorEnum)) + return + } + + imageName := dockerBuildInfo.Image + + // 判断本地是否已经有镜像了 + images, err := cli.ImageList(ctx, types.ImageListOptions{}) + if err != nil { + logs.Error("DOCKER_JOB|list docker images error ", err) + dockerBuildFinish(buildInfo.ToFinish(false, "获取docker镜像列表错误|"+err.Error(), api.DockerImagesFetchErrorEnum)) + return + } + localExist := false + imageStr := strings.TrimPrefix(strings.TrimPrefix(imageName, "http://"), "https://") + for _, image := range images { + for _, tagName := range image.RepoTags { + if tagName == imageStr { + localExist = true + } + } + } + + // 本地没有镜像的需要拉取新的镜像 + if !localExist { + postLog(false, "开始拉取镜像，镜像名称："+imageName, buildInfo) + postLog(false, "[提示]镜像比较大时，首次拉取时间会比较长。可以在构建机本地预先拉取镜像来提高流水线启动速度。", buildInfo) + reader, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{ + RegistryAuth: generateDockerAuth(dockerBuildInfo.Credential), + }) + if err != nil { + logs.Error(fmt.Sprintf("DOCKER_JOB|pull new image %s error ", imageName), err) + dockerBuildFinish(buildInfo.ToFinish(false, fmt.Sprintf("拉取镜像 %s 失败|%s", imageName, err.Error()), api.DockerImagePullErrorEnum)) + return + } + defer reader.Close() + buf := new(strings.Builder) + _, err = io.Copy(buf, reader) + if err != nil { + logs.Error("DOCKER_JOB|write image message error ", err) + postLog(true, "获取拉取镜像信息日志失败："+err.Error(), buildInfo) + } else { + // 异步打印，防止过大卡住主流程 + go postLog(false, buf.String(), buildInfo) + } + } else { + postLog(false, "本地存在镜像，准备启动构建环境..."+imageName, buildInfo) + } + + // 创建docker构建机运行准备空间，拉取docker构建机初始化文件 + tmpDir := fmt.Sprintf("%s/%s", systemutil.GetWorkDir(), localDockerBuildTmpDirName) + err = mkDir(tmpDir) + if err != nil { + errMsg := fmt.Sprintf("创建Docker构建临时目录失败(create tmp directory failed): %s", err.Error()) + logs.Error("DOCKER_JOB|" + errMsg) + dockerBuildFinish(buildInfo.ToFinish(false, errMsg, api.DockerMakeTmpDirErrorEnum)) + return + } + + // 创建容器 + containerName := fmt.Sprintf("dispatch-%s-%s-%s", buildInfo.BuildId, buildInfo.VmSeqId, util.RandStringRunes(8)) + mounts, err := parseContainerMounts(buildInfo) + if err != nil { + logs.Error("DOCKER_JOB| ", err) + dockerBuildFinish(buildInfo.ToFinish(false, err.Error(), api.DockerMountCreateErrorEnum)) + return + } + var resources container.Resources + if dockerBuildInfo.DockerResource != nil { + resources = container.Resources{ + Memory: dockerBuildInfo.DockerResource.MemoryLimitBytes, + CPUQuota: dockerBuildInfo.DockerResource.CpuQuota, + CPUPeriod: dockerBuildInfo.DockerResource.CpuPeriod, + } + } + hostConfig := &container.HostConfig{ + CapAdd: []string{"SYS_PTRACE"}, + Mounts: mounts, + NetworkMode: container.NetworkMode("bridge"), + Resources: resources, + } + + creatResp, err := cli.ContainerCreate(ctx, &container.Config{ + Image: imageStr, + Cmd: []string{}, + Entrypoint: []string{"/bin/sh", "-c", entryPointCmd}, + Env: parseContainerEnv(dockerBuildInfo), + }, hostConfig, nil, nil, containerName) + if err != nil { + logs.Error(fmt.Sprintf("DOCKER_JOB|create container %s error ", containerName), err) + dockerBuildFinish(buildInfo.ToFinish(false, fmt.Sprintf("创建容器 %s 失败|%s", containerName, err.Error()), api.DockerContainerCreateErrorEnum)) + return + } + + defer func() { + if err = cli.ContainerRemove(ctx, creatResp.ID, types.ContainerRemoveOptions{Force: true}); err != nil { + logs.Error(fmt.Sprintf("DOCKER_JOB|remove container %s error ", creatResp.ID), err) + } + }() + + // 启动容器 + if err := cli.ContainerStart(ctx, creatResp.ID, types.ContainerStartOptions{}); err != nil { + logs.Error(fmt.Sprintf("DOCKER_JOB|start container %s error ", creatResp.ID), err) + dockerBuildFinish(buildInfo.ToFinish(false, fmt.Sprintf("启动容器 %s 失败|%s", containerName, err.Error()), api.DockerContainerStartErrorEnum)) + return + } + + // 等待容器结束，处理错误信息并上报 + statusCh, errCh := cli.ContainerWait(ctx, creatResp.ID, container.WaitConditionNotRunning) + select { + case err := <-errCh: + if err != nil { + logs.Error(fmt.Sprintf("DOCKER_JOB|wait container %s over error ", creatResp.ID), err) + dockerBuildFinish(buildInfo.ToFinish(false, fmt.Sprintf("等待容器 %s 结束错误|%s", containerName, err.Error()), api.DockerContainerRunErrorEnum)) + return + } + case status := <-statusCh: + if status.Error != nil { + logs.Error(fmt.Sprintf("DOCKER_JOB|wait container %s over error ", creatResp.ID), status.Error) + dockerBuildFinish(buildInfo.ToFinish(false, fmt.Sprintf("等待容器 %s 结束错误|%s", containerName, status.Error.Message), api.DockerContainerRunErrorEnum)) + return + } else { + if status.StatusCode != 0 { + logs.Warn(fmt.Sprintf("DOCKER_JOB|wait container %s over status not 0, exit code %d", creatResp.ID, status.StatusCode)) + msg := "" + // 如果docker状态不为零，将日志上报 + logFile := func(tag string) (string, error) { + logsDir := fmt.Sprintf("%s/%s/logs/%s/%s", workDir, LocalDockerWorkSpaceDirName, buildInfo.BuildId, buildInfo.VmSeqId) + logFile := filepath.Join(logsDir, "docker.log") + content, err := os.ReadFile(logFile) + if err != nil && os.IsNotExist(err) { + return "", nil + } else if err != nil { + return "", err + } + // 超过字数说明肯定不是错误，不打印 + if len(content) > 1000 { + return tag, nil + } + return string(content), nil + } + content, err := logFile(longLogTag) + if err != nil { + msg = fmt.Sprintf("read log file error %s", err.Error()) + } else { + msg = content + } + // 这里可能就是docker最开始执行时报错，拿一下docker log + if msg == "" { + logs, err := cli.ContainerLogs(ctx, creatResp.ID, types.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) + if err != nil { + msg = "" + } else { + buf := new(strings.Builder) + _, err := io.Copy(buf, logs) + if err != nil { + msg = "" + } else { + msg = buf.String() + } + } + } + + if msg == longLogTag { + msg = "" + } + + dockerBuildFinish( + buildInfo.ToFinish(false, fmt.Sprintf( + "等待容器 %s 结束状态码为 %d 不为0 \n %s", + containerName, status.StatusCode, msg), + api.DockerContainerRunErrorEnum, + )) + return + } + } + } + + dockerBuildFinish(buildInfo.ToFinish(true, "", api.NoErrorEnum)) +} + +// dockerBuildFinish docker构建结束相关 +func dockerBuildFinish(buildInfo *api.ThirdPartyBuildWithStatus) { + if buildInfo == nil { + logs.Warn("DOCKER_JOB|buildInfo not exist") + return + } + + if buildInfo.Success { + time.Sleep(8 * time.Second) + } + result, err := api.WorkerBuildFinish(buildInfo) + if err != nil { + logs.Error("DOCKER_JOB|send worker build finish failed: ", err.Error()) + } + if result.IsNotOk() { + logs.Error("DOCKER_JOB|send worker build finish failed: ", result.Message) + } + logs.Info("DOCKER_JOB|workerBuildFinish done") +} + +// postLog 向后台上报日志 +func postLog(red bool, message string, buildInfo *api.ThirdPartyBuildInfo) { + taskId := "startVM-" + buildInfo.VmSeqId + + logMessage := &api.LogMessage{ + Message: message, + Timestamp: time.Now().UnixMilli(), + Tag: taskId, + JobId: buildInfo.ContainerHashId, + LogType: api.LogtypeLog, + ExecuteCount: buildInfo.ExecuteCount, + SubTag: nil, + } + + var err error + if red { + _, err = api.AddLogRedLine(buildInfo.BuildId, logMessage, buildInfo.VmSeqId) + } else { + _, err = api.AddLogLine(buildInfo.BuildId, logMessage, buildInfo.VmSeqId) + } + if err != nil { + logs.Error("DOCKER_JOB|api post log error", err) + } +} + +// mkDir 与 systemutil.MkBuildTmpDir 功能一致 +func mkDir(dir string) error { + err := os.MkdirAll(dir, os.ModePerm) + err2 := systemutil.Chmod(dir, os.ModePerm) + if err == nil && err2 != nil { + err = err2 + } + return err +} + +// generateDockerAuth 创建拉取docker凭据 +func generateDockerAuth(cred *api.Credential) string { + if cred == nil || cred.User == "" || cred.Password == "" { + return "" + } + + authConfig := types.AuthConfig{ + Username: cred.User, + Password: cred.Password, + } + encodedJSON, err := json.Marshal(authConfig) + if err != nil { + panic(err) + } + + return base64.URLEncoding.EncodeToString(encodedJSON) +} + +// parseContainerMounts 解析生成容器挂载内容 +func parseContainerMounts(buildInfo *api.ThirdPartyBuildInfo) ([]mount.Mount, error) { + var mounts []mount.Mount + + // 默认绑定本机的java用来执行worker，因为仅支持linux容器所以仅限linux构建机绑定 + if systemutil.IsLinux() { + javaDir := config.GAgentConfig.JdkDirPath + mounts = append(mounts, mount.Mount{ + Type: mount.TypeBind, + Source: javaDir, + Target: "/usr/local/jre", + ReadOnly: true, + }) + } + + // 挂载docker构建机初始化脚本 + workDir := systemutil.GetWorkDir() + mounts = append(mounts, mount.Mount{ + Type: mount.TypeBind, + Source: config.GetDockerInitFilePath(), + Target: entryPointCmd, + ReadOnly: true, + }) + + // 创建并挂载data和log + dataDir := fmt.Sprintf("%s/%s/data/%s/%s", workDir, LocalDockerWorkSpaceDirName, buildInfo.PipelineId, buildInfo.VmSeqId) + err := mkDir(dataDir) + if err != nil && !os.IsExist(err) { + return nil, errors.Wrapf(err, "create local data dir %s error", dataDir) + } + mounts = append(mounts, mount.Mount{ + Type: mount.TypeBind, + Source: dataDir, + Target: dockerDataDir, + ReadOnly: false, + }) + + logsDir := fmt.Sprintf("%s/%s/logs/%s/%s", workDir, LocalDockerWorkSpaceDirName, buildInfo.BuildId, buildInfo.VmSeqId) + err = mkDir(logsDir) + if err != nil && !os.IsExist(err) { + return nil, errors.Wrapf(err, "create local logs dir %s error", logsDir) + } + mounts = append(mounts, mount.Mount{ + Type: mount.TypeBind, + Source: logsDir, + Target: dockerLogDir, + ReadOnly: false, + }) + + return mounts, nil +} + +// parseContainerEnv 解析生成容器环境变量 +func parseContainerEnv(dockerBuildInfo *api.ThirdPartyDockerBuildInfo) []string { + var envs []string + + // 默认传入环境变量用来构建 + envs = append(envs, "devops_project_id="+config.GAgentConfig.ProjectId) + envs = append(envs, "devops_agent_id="+dockerBuildInfo.AgentId) + envs = append(envs, "devops_agent_secret_key="+dockerBuildInfo.SecretKey) + envs = append(envs, "devops_gateway="+config.GetGateWay()) + // 通过环境变量区分agent docker + envs = append(envs, "agent_build_env=DOCKER") + + if dockerBuildInfo.Envs == nil { + return envs + } + + for k, v := range dockerBuildInfo.Envs { + envs = append(envs, k+"="+v) + } + + return envs +} diff --git a/src/agent/src/pkg/job/build_manager.go b/src/agent/src/pkg/job/build_manager.go index 045cbc5fd4d..0c1ba62eef0 100644 --- a/src/agent/src/pkg/job/build_manager.go +++ b/src/agent/src/pkg/job/build_manager.go @@ -38,10 +38,8 @@ import ( "sync" ) -// buildManager 当前构建对象管理 +// buildManager 二进制构建对象管理 type buildManager struct { - // Lock 多协诚修改时的执行锁 - Lock sync.Mutex // preInstance 接取的构建任务但还没开始进行构建 [string]bool preInstances sync.Map // instances 正在执行中的构建对象 [int]*api.ThirdPartyBuildInfo @@ -99,7 +97,7 @@ func (b *buildManager) waitProcessDone(processId int) { errMsg := fmt.Sprintf("build process err, pid: %d, err: %s", processId, err.Error()) logs.Warn(errMsg) b.instances.Delete(processId) - workerBuildFinish(&api.ThirdPartyBuildWithStatus{ThirdPartyBuildInfo: *info, Message: errMsg}) + workerBuildFinish(info.ToFinish(false, errMsg, api.BuildProcessRunErrorEnum)) return } @@ -123,7 +121,11 @@ func (b *buildManager) waitProcessDone(processId int) { buildInfo := info b.instances.Delete(processId) - workerBuildFinish(&api.ThirdPartyBuildWithStatus{ThirdPartyBuildInfo: *buildInfo, Success: success, Message: msg}) + if success { + workerBuildFinish(buildInfo.ToFinish(success, msg, api.NoErrorEnum)) + } else { + workerBuildFinish(buildInfo.ToFinish(success, msg, api.BuildProcessRunErrorEnum)) + } } func (b *buildManager) GetPreInstancesCount() int { diff --git a/src/agent/src/pkg/job_docker/job_docker.go b/src/agent/src/pkg/job_docker/job_docker.go new file mode 100644 index 00000000000..3c2018f227a --- /dev/null +++ b/src/agent/src/pkg/job_docker/job_docker.go @@ -0,0 +1,23 @@ +package job_docker + +import ( + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" +) + +type DockerHostInfo struct { + ContainerCreateInfo ContainerCreateInfo +} + +type ImagePullInfo struct { + ImageName string + AuthType types.AuthConfig +} + +type ContainerCreateInfo struct { + ContainerName string + Config *container.Config + HostConfig *container.HostConfig + NetWorkingConfig *network.NetworkingConfig +} diff --git a/src/agent/src/pkg/pipeline/pipeline.go b/src/agent/src/pkg/pipeline/pipeline.go index 1ac1b0d8656..1c9f5994330 100644 --- a/src/agent/src/pkg/pipeline/pipeline.go +++ b/src/agent/src/pkg/pipeline/pipeline.go @@ -44,6 +44,12 @@ import ( ) func Start() { + defer func() { + if err := recover(); err != nil { + logs.Error("agent pipeline panic: ", err) + } + }() + time.Sleep(10 * time.Second) for { runPipeline() diff --git a/src/agent/src/pkg/upgrade/download/download_darwin.go b/src/agent/src/pkg/upgrade/download/download_darwin.go index 7d4ad9ed856..fb5ca701d6d 100644 --- a/src/agent/src/pkg/upgrade/download/download_darwin.go +++ b/src/agent/src/pkg/upgrade/download/download_darwin.go @@ -6,6 +6,7 @@ package download import ( "github.com/Tencent/bk-ci/src/agent/src/pkg/api" "github.com/Tencent/bk-ci/src/agent/src/pkg/config" + "github.com/pkg/errors" "runtime" "strings" ) @@ -43,3 +44,7 @@ func DownloadJdkFile(saveDir string) (string, error) { "jre/"+strings.TrimPrefix(getServerFileArch(), "_")+"/jre.zip", saveDir+"/"+config.JdkClientFile, ) } + +func DownloadDockerInitFile(saveDir string) (string, error) { + return "", errors.New("not support macos use docker agent") +} diff --git a/src/agent/src/pkg/upgrade/download/download_unix.go b/src/agent/src/pkg/upgrade/download/download_unix.go index 8040d922139..34bd44a23cd 100644 --- a/src/agent/src/pkg/upgrade/download/download_unix.go +++ b/src/agent/src/pkg/upgrade/download/download_unix.go @@ -45,3 +45,9 @@ func DownloadJdkFile(saveDir string) (string, error) { "jre/"+strings.TrimPrefix(getServerFileArch(), "_")+"/jre.zip", saveDir+"/"+config.JdkClientFile, ) } + +func DownloadDockerInitFile(saveDir string) (string, error) { + return api.DownloadUpgradeFile( + "script/linux/agent_docker_init.sh", saveDir+"/"+config.DockerInitFile, + ) +} diff --git a/src/agent/src/pkg/upgrade/download/download_win.go b/src/agent/src/pkg/upgrade/download/download_win.go index 58887e1660e..e1e35f3fe8b 100644 --- a/src/agent/src/pkg/upgrade/download/download_win.go +++ b/src/agent/src/pkg/upgrade/download/download_win.go @@ -6,6 +6,7 @@ package download import ( "github.com/Tencent/bk-ci/src/agent/src/pkg/api" "github.com/Tencent/bk-ci/src/agent/src/pkg/config" + "github.com/pkg/errors" ) func DownloadUpgradeFile(saveDir string) (string, error) { @@ -31,3 +32,7 @@ func DownloadJdkFile(saveDir string) (string, error) { "jre/windows/jre.zip", saveDir+"/"+config.JdkClientFile, ) } + +func DownloadDockerInitFile(saveDir string) (string, error) { + return "", errors.New("not support windows use docker agent") +} diff --git a/src/agent/src/pkg/upgrade/operation.go b/src/agent/src/pkg/upgrade/operation.go index a66d99e996f..edb06c6e175 100644 --- a/src/agent/src/pkg/upgrade/operation.go +++ b/src/agent/src/pkg/upgrade/operation.go @@ -63,7 +63,7 @@ func runUpgrader(action string) error { scripPath := systemutil.GetUpgradeDir() + "/" + config.GetClientUpgraderFile() if !systemutil.IsWindows() { - err := os.Chmod(scripPath, 0777) + err := systemutil.Chmod(scripPath, 0777) if err != nil { logs.Error("[agentUpgrade]|chmod failed: ", err.Error()) return errors.New("chmod failed: ") @@ -88,24 +88,30 @@ func runUpgrader(action string) error { } // DoUpgradeOperation 调用升级程序 -func DoUpgradeOperation(agentChanged bool, workAgentChanged bool, jdkChanged bool) error { - logs.Info("[agentUpgrade]|start upgrade, agent changed: ", agentChanged, ", work agent changed: ", workAgentChanged, ", jdk agent changed: ", jdkChanged) - - if !agentChanged && !workAgentChanged && !jdkChanged { +func DoUpgradeOperation(changeItems upgradeChangeItem) error { + logs.Info("[agentUpgrade]|start upgrade, agent changed: ", changeItems.AgentChanged, + ", work agent changed: ", changeItems.WorkAgentChanged, + ", jdk agent changed: ", changeItems.JdkChanged, + ", docker init file changed: ", changeItems.DockerInitFile, + ) + + if changeItems.checkNoChange() { logs.Info("[agentUpgrade]|no change to upgrade, skip") return nil } // 进入升级逻辑时防止agent接构建任务，同时确保无任何构建任务在进行 - job.GBuildManager.Lock.Lock() + job.BuildTotalManager.Lock.Lock() defer func() { - job.GBuildManager.Lock.Unlock() + job.BuildTotalManager.Lock.Unlock() }() - if job.GBuildManager.GetPreInstancesCount() > 0 && job.GBuildManager.GetInstanceCount() > 0 { + if job.GBuildManager.GetPreInstancesCount() > 0 || job.GBuildManager.GetInstanceCount() > 0 || + job.GBuildDockerManager.GetInstanceCount() > 0 { + logs.Info("agent has upgrade item, but has job running, so skip.") return nil } - if jdkChanged { + if changeItems.JdkChanged { logs.Info("[agentUpgrade]|jdk changed, replace jdk file") workDir := systemutil.GetWorkDir() @@ -148,9 +154,11 @@ func DoUpgradeOperation(agentChanged bool, workAgentChanged bool, jdkChanged boo config.SaveJdkDir(workDir + "/" + jdkTmpName) logs.Info("[agentUpgrade]|replace jdk file done") + } else { + logs.Info("[agentUpgrade]|jdk not changed, skip agent upgrade") } - if workAgentChanged { + if changeItems.WorkAgentChanged { logs.Info("[agentUpgrade]|work agent changed, replace work agent file") _, err := fileutil.CopyFile( systemutil.GetUpgradeDir()+"/"+config.WorkAgentFile, @@ -163,9 +171,11 @@ func DoUpgradeOperation(agentChanged bool, workAgentChanged bool, jdkChanged boo logs.Info("[agentUpgrade]|replace agent file done") config.GAgentEnv.SlaveVersion = config.DetectWorkerVersion() + } else { + logs.Info("[agentUpgrade]|worker not changed, skip agent upgrade") } - if agentChanged { + if changeItems.AgentChanged { logs.Info("[agentUpgrade]|agent changed, start upgrader") err := runUpgrader(config.ActionUpgrade) if err != nil { @@ -175,5 +185,26 @@ func DoUpgradeOperation(agentChanged bool, workAgentChanged bool, jdkChanged boo logs.Info("[agentUpgrade]|agent not changed, skip agent upgrade") } + if changeItems.DockerInitFile { + logs.Info("[agentUpgrade]|docker init file changed, replace docker init file") + _, err := fileutil.CopyFile( + systemutil.GetUpgradeDir()+"/"+config.DockerInitFile, + config.GetDockerInitFilePath(), + true) + if err != nil { + logs.Error("[agentUpgrade]|replace work docker init file failed: ", err.Error()) + return errors.New("replace work docker init file failed") + } + // 授予文件可执行权限，每次升级后赋予权限可以减少直接在启动时赋予的并发赋予的情况 + if err = systemutil.Chmod(config.GetDockerInitFilePath(), os.ModePerm); err != nil { + logs.Error("[agentUpgrade]|chmod work docker init file failed: ", err.Error()) + return errors.New("chmod work docker init file failed") + } + + logs.Info("[agentUpgrade]|replace docker init file done") + } else { + logs.Info("[agentUpgrade]|docker init file not changed, skip agent upgrade") + } + return nil } diff --git a/src/agent/src/pkg/upgrade/upgrade.go b/src/agent/src/pkg/upgrade/upgrade.go index 0dd53ef7711..9603b9c0d77 100644 --- a/src/agent/src/pkg/upgrade/upgrade.go +++ b/src/agent/src/pkg/upgrade/upgrade.go @@ -34,6 +34,7 @@ import ( "github.com/pkg/errors" "os" "strings" + "sync" "sync/atomic" "time" @@ -44,15 +45,63 @@ import ( "github.com/Tencent/bk-ci/src/agent/src/pkg/util/systemutil" ) +var JdkVersion = &JdkVersionType{} + // JdkVersion jdk版本信息缓存 -var JdkVersion struct { +type JdkVersionType struct { JdkFileModTime time.Time // 版本信息，原子级的 []string - Version atomic.Value + version atomic.Value +} + +func (j *JdkVersionType) GetVersion() []string { + data := j.version.Load() + if data == nil { + return []string{} + } else { + return j.version.Load().([]string) + } +} + +func (j *JdkVersionType) SetVersion(version []string) { + if version == nil { + version = []string{} + } + j.version.Swap(version) +} + +// DockerFileMd5 缓存，用来计算md5 +var DockerFileMd5 struct { + // 目前非linux机器不支持，以及一些机器不使用docker就不用计算md5 + NeedUpgrade bool + FileModTime time.Time + Lock sync.Mutex + Md5 string +} + +type upgradeChangeItem struct { + AgentChanged bool + WorkAgentChanged bool + JdkChanged bool + DockerInitFile bool +} + +func (u upgradeChangeItem) checkNoChange() bool { + if !u.AgentChanged && !u.WorkAgentChanged && !u.JdkChanged && !u.DockerInitFile { + return true + } + + return false } // DoPollAndUpgradeAgent 循环，每20s一次执行升级 func DoPollAndUpgradeAgent() { + defer func() { + if err := recover(); err != nil { + logs.Error("agent upgrade panic: ", err) + } + }() + for { time.Sleep(20 * time.Second) logs.Info("try upgrade") @@ -78,7 +127,17 @@ func agentUpgrade() { logs.Error("[agentUpgrade]|sync jdk version err: ", err.Error()) return } - checkResult, err := api.CheckUpgrade(jdkVersion) + + err = syncDockerInitFileMd5() + if err != nil { + logs.Error("[agentUpgrade]|sync docker file md5 err: ", err.Error()) + return + } + + checkResult, err := api.CheckUpgrade(jdkVersion, api.DockerInitFileInfo{ + FileMd5: DockerFileMd5.Md5, + NeedUpgrade: DockerFileMd5.NeedUpgrade, + }) if err != nil { ack = true logs.Error("[agentUpgrade]|check upgrade err: ", err.Error()) @@ -96,20 +155,20 @@ func agentUpgrade() { upgradeItem := new(api.UpgradeItem) err = util.ParseJsonToData(checkResult.Data, &upgradeItem) - if !upgradeItem.Agent && !upgradeItem.Worker && !upgradeItem.Jdk { + if !upgradeItem.Agent && !upgradeItem.Worker && !upgradeItem.Jdk && !upgradeItem.DockerInitFile { logs.Info("[agentUpgrade]|no need to upgrade agent, skip") return } ack = true logs.Info("[agentUpgrade]|download upgrade files start") - agentChanged, workerChanged, jdkChanged := downloadUpgradeFiles(upgradeItem) - if !agentChanged && !workerChanged && !jdkChanged { + changeItems := downloadUpgradeFiles(upgradeItem) + if changeItems.checkNoChange() { return } logs.Info("[agentUpgrade]|download upgrade files done") - err = DoUpgradeOperation(agentChanged, workerChanged, jdkChanged) + err = DoUpgradeOperation(changeItems) if err != nil { logs.Error("[agentUpgrade]|do upgrade operation failed", err) } else { @@ -125,7 +184,7 @@ func syncJdkVersion() ([]string, error) { if os.IsNotExist(err) { logs.Error("syncJdkVersion no jdk dir find", err) // jdk版本置为空，否则会一直保持有版本的状态 - JdkVersion.Version.Swap(nil) + JdkVersion.SetVersion([]string{}) return nil, nil } return nil, errors.Wrap(err, "agent check jdk dir error") @@ -133,35 +192,82 @@ func syncJdkVersion() ([]string, error) { nowModTime := stat.ModTime() // 如果为空则必获取 - if JdkVersion.Version.Load() == nil { + if len(JdkVersion.GetVersion()) == 0 { version, err := getJdkVersion() if err != nil { // 拿取错误时直接下载新的 logs.Error("syncJdkVersion getJdkVersion err", err) return nil, nil } - JdkVersion.Version.Swap(version) + JdkVersion.SetVersion(version) JdkVersion.JdkFileModTime = nowModTime return version, nil } // 判断文件夹最后修改时间，不一致时不用更改 if nowModTime == JdkVersion.JdkFileModTime { - return JdkVersion.Version.Load().([]string), nil + return JdkVersion.GetVersion(), nil } version, err := getJdkVersion() if err != nil { // 拿取错误时直接下载新的 logs.Error("syncJdkVersion getJdkVersion err", err) - JdkVersion.Version.Swap(nil) + JdkVersion.SetVersion([]string{}) return nil, nil } - JdkVersion.Version.Swap(version) + JdkVersion.SetVersion(version) JdkVersion.JdkFileModTime = nowModTime return version, nil } +func syncDockerInitFileMd5() error { + if !systemutil.IsLinux() || !config.GAgentConfig.EnableDockerBuild { + DockerFileMd5.NeedUpgrade = false + return nil + } + DockerFileMd5.Lock.Lock() + defer func() { + DockerFileMd5.Lock.Unlock() + }() + DockerFileMd5.NeedUpgrade = true + + filePath := config.GetDockerInitFilePath() + + stat, err := os.Stat(filePath) + if err != nil { + if os.IsNotExist(err) { + logs.Warn("syncDockerInitFileMd5 no docker init file find", err) + DockerFileMd5.Md5 = "" + return nil + } + return errors.Wrap(err, "agent check docker init file error") + } + nowModTime := stat.ModTime() + + if DockerFileMd5.Md5 == "" { + DockerFileMd5.Md5, err = fileutil.GetFileMd5(filePath) + if err != nil { + DockerFileMd5.Md5 = "" + return errors.Wrapf(err, "agent get docker init file %s md5 error", filePath) + } + DockerFileMd5.FileModTime = nowModTime + return nil + } + + if nowModTime == DockerFileMd5.FileModTime { + return nil + } + + DockerFileMd5.Md5, err = fileutil.GetFileMd5(filePath) + if err != nil { + DockerFileMd5.Md5 = "" + return errors.Wrapf(err, "agent get docker init file %s md5 error", filePath) + } + DockerFileMd5.FileModTime = nowModTime + return nil +} + func getJdkVersion() ([]string, error) { jdkVersion, err := command.RunCommand(config.GetJava(), []string{"-version"}, "", nil) if err != nil { @@ -170,40 +276,92 @@ func getJdkVersion() ([]string, error) { } var jdkV []string if jdkVersion != nil { - jdkV = strings.Split(strings.TrimSuffix(strings.TrimSpace(string(jdkVersion)), "\n"), "\n") - for i, j := range jdkV { - jdkV[i] = strings.TrimSpace(j) - } + versionOutputString := strings.TrimSpace(string(jdkVersion)) + jdkV = trimJdkVersionList(versionOutputString) } return jdkV, nil } +// parseJdkVersionList 清洗在解析一些版本信息的干扰信息,避免因tmp空间满等导致识别不准确造成重复不断的升级 +func trimJdkVersionList(versionOutputString string) []string { + /* + OpenJDK 64-Bit Server VM warning: Insufficient space for shared memory file: + 32490 + Try using the -Djava.io.tmpdir= option to select an alternate temp location. + + openjdk version "1.8.0_352" + OpenJDK Runtime Environment (Tencent Kona 8.0.12) (build 1.8.0_352-b1) + OpenJDK 64-Bit Server VM (Tencent Kona 8.0.12) (build 25.352-b1, mixed mode) + Picked up _JAVA_OPTIONS: -Xmx8192m -Xms256m -Xss8m + */ + // 一个JVM版本只需要识别3行。 + var jdkV = make([]string, 3) + + var sep = "\n" + if strings.HasSuffix(versionOutputString, "\r\n") { + sep = "\r\n" + } + + lines := strings.Split(strings.TrimSuffix(versionOutputString, sep), sep) + + var pos = 0 + for i := range lines { + + if pos == 0 { + if strings.Contains(lines[i], " version ") { + jdkV[pos] = lines[i] + pos++ + } + } else if pos == 1 { + if strings.Contains(lines[i], " Runtime Environment ") { + jdkV[pos] = lines[i] + pos++ + } + } else if pos == 2 { + if strings.Contains(lines[i], " Server VM ") { + jdkV[pos] = lines[i] + break + } + } + } + + return jdkV +} + // downloadUpgradeFiles 下载升级文件 -func downloadUpgradeFiles(item *api.UpgradeItem) (agentChanged, workAgentChanged, jdkChanged bool) { +func downloadUpgradeFiles(item *api.UpgradeItem) upgradeChangeItem { workDir := systemutil.GetWorkDir() upgradeDir := systemutil.GetUpgradeDir() _ = os.MkdirAll(upgradeDir, os.ModePerm) + result := upgradeChangeItem{} + if !item.Agent { - agentChanged = false + result.AgentChanged = false } else { - agentChanged = downloadUpgradeAgent(workDir, upgradeDir) + result.AgentChanged = downloadUpgradeAgent(workDir, upgradeDir) } if !item.Worker { - workAgentChanged = false + result.WorkAgentChanged = false } else { - workAgentChanged = downloadUpgradeWorker(workDir, upgradeDir) + result.WorkAgentChanged = downloadUpgradeWorker(workDir, upgradeDir) } if !item.Jdk { - jdkChanged = false + result.JdkChanged = false + } else { + result.JdkChanged = downloadUpgradeJdk(upgradeDir) + } + + if !item.DockerInitFile { + result.DockerInitFile = false } else { - jdkChanged = downloadUpgradeJdk(upgradeDir) + result.DockerInitFile = downloadUpgradeDockerInit(upgradeDir) } - return agentChanged, workAgentChanged, jdkChanged + return result } func downloadUpgradeAgent(workDir, upgradeDir string) (agentChanged bool) { @@ -293,3 +451,15 @@ func downloadUpgradeJdk(upgradeDir string) (jdkChanged bool) { return true } + +func downloadUpgradeDockerInit(upgradeDir string) bool { + logs.Info("[agentUpgrade]|download docker init shell start") + _, err := download.DownloadDockerInitFile(upgradeDir) + if err != nil { + logs.Error("[agentUpgrade]|download docker init shell failed", err) + return false + } + logs.Info("[agentUpgrade]|download docker init shell done") + + return true +} diff --git a/src/agent/src/pkg/upgrade/upgrade_test.go b/src/agent/src/pkg/upgrade/upgrade_test.go new file mode 100644 index 00000000000..01e795205e9 --- /dev/null +++ b/src/agent/src/pkg/upgrade/upgrade_test.go @@ -0,0 +1,61 @@ +package upgrade + +import ( + "reflect" + "testing" +) + +func Test_trimJdkVersionList(t *testing.T) { + want := []string{ + "openjdk version \"1.8.0_352\"", + "OpenJDK Runtime Environment (Tencent Kona 8.0.12) (build 1.8.0_352-b1)", + "OpenJDK 64-Bit Server VM (Tencent Kona 8.0.12) (build 25.352-b1, mixed mode)", + } + + sep := "\r\n" + jdkVersionString := "OpenJDK 64-Bit Server VM warning: Insufficient space for shared memory file:" + sep + + " 9507" + sep + + "Try using the -Djava.io.tmpdir= option to select an alternate temp location." + sep + sep + + want[0] + sep + + want[1] + sep + + want[2] + sep + + "Picked up _JAVA_OPTIONS: -Xmx8192m -Xms256m -Xss8m" + sep + + loopTest(t, "windows_un_normal", jdkVersionString, want) + + jdkVersionString = want[0] + sep + want[1] + sep + want[2] + sep + + loopTest(t, "windows_normal", jdkVersionString, want) + + sep = "\n" + jdkVersionString = "OpenJDK 64-Bit Server VM warning: Insufficient space for shared memory file:" + sep + + " 9507" + sep + + "Try using the -Djava.io.tmpdir= option to select an alternate temp location." + sep + sep + + want[0] + sep + + want[1] + sep + + want[2] + sep + + "Picked up _JAVA_OPTIONS: -Xmx8192m -Xms256m -Xss8m" + sep + + loopTest(t, "linux_un_normal", jdkVersionString, want) + + jdkVersionString = want[0] + sep + want[1] + sep + want[2] + sep + + loopTest(t, "linux_normal", jdkVersionString, want) +} + +func loopTest(t *testing.T, name string, jdkVersionString string, want []string) { + tests := trimJdkVersionList(jdkVersionString) + t.Run(name+"|length=3", func(t *testing.T) { + if len(tests) != len(want) { + t.Fatalf("\nFail: len(%d), want(%d)", len(tests), len(want)) + } + }) + + for i := 0; i < len(tests); i++ { + t.Run(name+"|"+tests[i], func(t *testing.T) { + if !reflect.DeepEqual(tests[i], want[i]) { + t.Fatalf("\nFail: %v \nWant: %v", tests[i], want[i]) + } + }) + } +} diff --git a/src/agent/src/pkg/util/httputil/devops.go b/src/agent/src/pkg/util/httputil/devops.go index 02fe788c8ab..042184fd472 100644 --- a/src/agent/src/pkg/util/httputil/devops.go +++ b/src/agent/src/pkg/util/httputil/devops.go @@ -30,16 +30,18 @@ package httputil import ( "encoding/json" "errors" - "github.com/Tencent/bk-ci/src/agent/internal/third_party/dep/fs" "io" "io/ioutil" "net/http" "os" "path/filepath" + "github.com/Tencent/bk-ci/src/agent/internal/third_party/dep/fs" + "github.com/Tencent/bk-ci/src/agent/src/pkg/config" "github.com/Tencent/bk-ci/src/agent/src/pkg/logs" "github.com/Tencent/bk-ci/src/agent/src/pkg/util/fileutil" + "github.com/Tencent/bk-ci/src/agent/src/pkg/util/systemutil" ) type DevopsResult struct { @@ -228,8 +230,8 @@ func AtomicWriteFile(filename string, reader io.Reader, mode os.FileMode) error if err := tempFile.Close(); err != nil { return err } - - if err := os.Chmod(tempName, mode); err != nil { + + if err := systemutil.Chmod(tempName, mode); err != nil { return err } diff --git a/src/agent/src/pkg/util/systemutil/dir_operation.go b/src/agent/src/pkg/util/systemutil/dir_operation.go index ef319584a21..d712144bfd8 100644 --- a/src/agent/src/pkg/util/systemutil/dir_operation.go +++ b/src/agent/src/pkg/util/systemutil/dir_operation.go @@ -61,7 +61,7 @@ func Chmod(file string, perm os.FileMode) error { if err == nil { logs.Info("chmod %o %s ok!", perm, file) } else { - logs.Warn("chmod %o %s msg: %s", perm, file, err) + logs.Warn("chmod %o %s msg: %s", perm, file, err.Error()) } return err } diff --git a/src/agent/src/pkg/util/systemutil/dir_operation_win.go b/src/agent/src/pkg/util/systemutil/dir_operation_win.go index 20e3a55db5a..45c0410a3fb 100644 --- a/src/agent/src/pkg/util/systemutil/dir_operation_win.go +++ b/src/agent/src/pkg/util/systemutil/dir_operation_win.go @@ -43,8 +43,16 @@ func MkBuildTmpDir() (string, error) { return tmpDir, err } -// Chmod windows 暂时不做任何事情 +// Chmod windows go的win实现只有 0400 只读和 0600 读写的区分，所以这里暂时先和0666对比 func Chmod(file string, perm os.FileMode) error { - logs.Info("chmod %d %s do nothing in windows", perm, file) - return nil + stat, err := os.Stat(file) + if stat != nil && stat.Mode() != 0666 { + err = os.Chmod(file, perm) + } + if err == nil { + logs.Info("chmod %o %s ok!", perm, file) + } else { + logs.Warn("chmod %o %s msg: %s", perm, file, err.Error()) + } + return err } diff --git a/src/agent/src/pkg/util/util.go b/src/agent/src/pkg/util/util.go index e8e01e69fc7..f5a8faaa54c 100644 --- a/src/agent/src/pkg/util/util.go +++ b/src/agent/src/pkg/util/util.go @@ -29,6 +29,7 @@ package util import ( "encoding/json" + "math/rand" "strings" "time" ) @@ -72,3 +73,13 @@ func Contains(s []string, subs string) bool { } return false } + +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") + +func RandStringRunes(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} diff --git a/src/agent/src/pkg/util/util_test.go b/src/agent/src/pkg/util/util_test.go index b9861ba9987..8aac9415d13 100644 --- a/src/agent/src/pkg/util/util_test.go +++ b/src/agent/src/pkg/util/util_test.go @@ -95,3 +95,27 @@ func TestContains(t *testing.T) { }) } } + +func TestRandStringRunes(t *testing.T) { + type args struct { + n int + } + tests := []struct { + name string + args args + want string + }{ + { + name: "测试随机字符串8位", + args: args{n: 8}, + want: "abcdefgh", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := RandStringRunes(tt.args.n); len(got) != len(tt.want) { + t.Errorf("RandStringRunes() = %v, want %v", len(got), len(tt.want)) + } + }) + } +} diff --git a/src/backend/booster/bk_dist/booster/command/command.go b/src/backend/booster/bk_dist/booster/command/command.go index 77774e910a7..8f0a552a217 100644 --- a/src/backend/booster/bk_dist/booster/command/command.go +++ b/src/backend/booster/bk_dist/booster/command/command.go @@ -74,6 +74,8 @@ const ( FlagPumpCacheDir = "pump_cache_dir" FlagPumpCacheSizeMaxMB = "pump_cache_size_max_MB" FlagPumpCacheRemoveAll = "pump_cache_remove_all" + FlagPumpBlackList = "pump_black_list" + FlagPumpMinActionNum = "pump_min_action_num" FlagForceLocalList = "force_local_list" FlagNoWork = "no_work" FlagControllerNoWait = "controller_no_wait" @@ -306,6 +308,14 @@ var ( Name: "pump_cache_remove_all", Usage: "remove all of pump cache files", }, + commandCli.StringSliceFlag{ + Name: "pump_black_list, pbl", + Usage: "action in this list will not use pump", + }, + commandCli.IntFlag{ + Name: "pump_min_action_num", + Usage: "do not use pump if total actions less this", + }, commandCli.StringSliceFlag{ Name: "force_local_list, fll", Usage: "key list which will be force executed locally", diff --git a/src/backend/booster/bk_dist/booster/command/process.go b/src/backend/booster/bk_dist/booster/command/process.go index e367461a4a6..9229fadd58a 100644 --- a/src/backend/booster/bk_dist/booster/command/process.go +++ b/src/backend/booster/bk_dist/booster/command/process.go @@ -155,6 +155,7 @@ func newBooster(c *commandCli.Context) (*pkg.Booster, error) { usr, err := user.Current() if err != nil { blog.Warnf("booster-command: get current user failed: %v", err) + return nil, err } // decide which server to connect to. @@ -220,6 +221,11 @@ func newBooster(c *commandCli.Context) (*pkg.Booster, error) { pumpCacheSizeMaxMB = 1024 } + pumpMinActionNum := 50 + if c.IsSet(FlagPumpMinActionNum) { + pumpMinActionNum = c.Int(FlagPumpMinActionNum) + } + // generate a new booster. cmdConfig := dcType.BoosterConfig{ Type: dcType.GetBoosterType(bt), @@ -273,6 +279,8 @@ func newBooster(c *commandCli.Context) (*pkg.Booster, error) { PumpCacheDir: c.String(FlagPumpCacheDir), PumpCacheSizeMaxMB: pumpCacheSizeMaxMB, PumpCacheRemoveAll: c.Bool(FlagPumpCacheRemoveAll), + PumpBlackList: c.StringSlice(FlagPumpBlackList), + PumpMinActionNum: int32(pumpMinActionNum), ForceLocalList: c.StringSlice(FlagForceLocalList), NoWork: c.Bool(FlagNoWork), WriteMemroy: c.Bool(FlagWriteMemroMemroy), diff --git a/src/backend/booster/bk_dist/booster/pkg/booster.go b/src/backend/booster/bk_dist/booster/pkg/booster.go index 1da13ec87f0..d0d9c7657ef 100644 --- a/src/backend/booster/bk_dist/booster/pkg/booster.go +++ b/src/backend/booster/bk_dist/booster/pkg/booster.go @@ -301,6 +301,11 @@ func (b *Booster) getWorkersEnv() map[string]string { requiredEnv[env.KeyExecutorPumpCacheDir] = b.config.Works.PumpCacheDir requiredEnv[env.KeyExecutorPumpCacheSizeMaxMB] = strconv.Itoa(int(b.config.Works.PumpCacheSizeMaxMB)) + if len(b.config.Works.PumpBlackList) > 0 { + requiredEnv[env.KeyExecutorPumpBlackKeys] = strings.Join(b.config.Works.PumpBlackList, env.CommonBKEnvSepKey) + } + requiredEnv[env.KeyExecutorPumpMinActionNum] = strconv.Itoa(int(b.config.Works.PumpMinActionNum)) + if b.config.Works.IOTimeoutSecs > 0 { requiredEnv[env.KeyExecutorIOTimeout] = strconv.Itoa(b.config.Works.IOTimeoutSecs) } diff --git a/src/backend/booster/bk_dist/common/env/env.go b/src/backend/booster/bk_dist/common/env/env.go index f2ee8db6ff3..b0b577e6438 100644 --- a/src/backend/booster/bk_dist/common/env/env.go +++ b/src/backend/booster/bk_dist/common/env/env.go @@ -46,12 +46,15 @@ const ( KeyExecutorPumpCache = "PUMP_CACHE" // cache pump inlude files KeyExecutorPumpCacheDir = "PUMP_CACHE_DIR" // cache pump inlude files KeyExecutorPumpCacheSizeMaxMB = "PUMP_CACHE_SIZE_MAX_MB" // cache pump inlude files + KeyExecutorPumpBlackKeys = "PUMP_BLACK_KEYS" + KeyExecutorPumpMinActionNum = "PUMP_MIN_ACTION_NUM" KeyExecutorForceLocalKeys = "FORCE_LOCAL_KEYS" KeyExecutorEnvProfile = "ENV_PROFILE" KeyExecutorWorkerSideCache = "WORKER_SIDE_CACHE" KeyExecutorLocalRecord = "LOCAL_RECORD" KeyExecutorWriteMemory = "WRITE_MEMORY" KeyExecutorIdleKeepSecs = "IDLE_KEEP_SECS" + KeyExecutorTotalActionNum = "TOTAL_ACTION_NUM" KeyUserDefinedLogLevel = "USER_DEFINED_LOG_LEVEL" KeyUserDefinedExecutorLogLevel = "USER_DEFINED_EXECUTOR_LOG_LEVEL" diff --git a/src/backend/booster/bk_dist/common/file/file.go b/src/backend/booster/bk_dist/common/file/file.go index 06273319967..85ab6b827f1 100644 --- a/src/backend/booster/bk_dist/common/file/file.go +++ b/src/backend/booster/bk_dist/common/file/file.go @@ -66,11 +66,11 @@ func (i *Info) Exist() bool { } // ModifyTime return the nano-second of this file's mod time -func (i *Info) ModifyTime() int { +func (i *Info) ModifyTime() int64 { if i.info == nil { return 0 } - return i.info.ModTime().Nanosecond() + return i.info.ModTime().UnixNano() } // ModifyTime64 return the ModifyTime as int64 diff --git a/src/backend/booster/bk_dist/common/protocol/task_worker.pb.go b/src/backend/booster/bk_dist/common/protocol/task_worker.pb.go index 60bd7128c59..8a630da4ef2 100644 --- a/src/backend/booster/bk_dist/common/protocol/task_worker.pb.go +++ b/src/backend/booster/bk_dist/common/protocol/task_worker.pb.go @@ -54,7 +54,7 @@ func (x *PBCompressType) UnmarshalJSON(data []byte) error { return nil } func (PBCompressType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{0} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{0} } type PBCacheStatus int32 @@ -96,7 +96,7 @@ func (x *PBCacheStatus) UnmarshalJSON(data []byte) error { return nil } func (PBCacheStatus) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{1} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{1} } type PBCmdType int32 @@ -153,7 +153,7 @@ func (x *PBCmdType) UnmarshalJSON(data []byte) error { return nil } func (PBCmdType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{2} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{2} } type PBFileDesc struct { @@ -167,6 +167,9 @@ type PBFileDesc struct { Targetrelativepath *string `protobuf:"bytes,7,opt,name=targetrelativepath" json:"targetrelativepath,omitempty"` Filemode *uint32 `protobuf:"varint,8,opt,name=filemode" json:"filemode,omitempty"` Linktarget []byte `protobuf:"bytes,9,opt,name=linktarget" json:"linktarget,omitempty"` + Modifytime *int64 `protobuf:"varint,10,opt,name=modifytime" json:"modifytime,omitempty"` + Accesstime *int64 `protobuf:"varint,11,opt,name=accesstime" json:"accesstime,omitempty"` + Createtime *int64 `protobuf:"varint,12,opt,name=createtime" json:"createtime,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -176,7 +179,7 @@ func (m *PBFileDesc) Reset() { *m = PBFileDesc{} } func (m *PBFileDesc) String() string { return proto.CompactTextString(m) } func (*PBFileDesc) ProtoMessage() {} func (*PBFileDesc) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{0} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{0} } func (m *PBFileDesc) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBFileDesc.Unmarshal(m, b) @@ -259,6 +262,27 @@ func (m *PBFileDesc) GetLinktarget() []byte { return nil } +func (m *PBFileDesc) GetModifytime() int64 { + if m != nil && m.Modifytime != nil { + return *m.Modifytime + } + return 0 +} + +func (m *PBFileDesc) GetAccesstime() int64 { + if m != nil && m.Accesstime != nil { + return *m.Accesstime + } + return 0 +} + +func (m *PBFileDesc) GetCreatetime() int64 { + if m != nil && m.Createtime != nil { + return *m.Createtime + } + return 0 +} + type PBFileResult struct { Fullpath *string `protobuf:"bytes,1,req,name=fullpath" json:"fullpath,omitempty"` Retcode *int32 `protobuf:"varint,2,req,name=retcode" json:"retcode,omitempty"` @@ -272,7 +296,7 @@ func (m *PBFileResult) Reset() { *m = PBFileResult{} } func (m *PBFileResult) String() string { return proto.CompactTextString(m) } func (*PBFileResult) ProtoMessage() {} func (*PBFileResult) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{1} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{1} } func (m *PBFileResult) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBFileResult.Unmarshal(m, b) @@ -330,7 +354,7 @@ func (m *PBCommand) Reset() { *m = PBCommand{} } func (m *PBCommand) String() string { return proto.CompactTextString(m) } func (*PBCommand) ProtoMessage() {} func (*PBCommand) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{2} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{2} } func (m *PBCommand) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBCommand.Unmarshal(m, b) @@ -411,7 +435,7 @@ func (m *PBStatEntry) Reset() { *m = PBStatEntry{} } func (m *PBStatEntry) String() string { return proto.CompactTextString(m) } func (*PBStatEntry) ProtoMessage() {} func (*PBStatEntry) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{3} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{3} } func (m *PBStatEntry) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBStatEntry.Unmarshal(m, b) @@ -461,7 +485,7 @@ func (m *PBResult) Reset() { *m = PBResult{} } func (m *PBResult) String() string { return proto.CompactTextString(m) } func (*PBResult) ProtoMessage() {} func (*PBResult) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{4} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{4} } func (m *PBResult) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBResult.Unmarshal(m, b) @@ -529,6 +553,11 @@ type PBCacheParam struct { Size *int64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"` Target []byte `protobuf:"bytes,4,req,name=target" json:"target,omitempty"` Overwrite *int32 `protobuf:"varint,5,opt,name=overwrite" json:"overwrite,omitempty"` + Filemode *uint32 `protobuf:"varint,6,opt,name=filemode" json:"filemode,omitempty"` + Linktarget []byte `protobuf:"bytes,7,opt,name=linktarget" json:"linktarget,omitempty"` + Modifytime *int64 `protobuf:"varint,8,opt,name=modifytime" json:"modifytime,omitempty"` + Accesstime *int64 `protobuf:"varint,9,opt,name=accesstime" json:"accesstime,omitempty"` + Createtime *int64 `protobuf:"varint,10,opt,name=createtime" json:"createtime,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -538,7 +567,7 @@ func (m *PBCacheParam) Reset() { *m = PBCacheParam{} } func (m *PBCacheParam) String() string { return proto.CompactTextString(m) } func (*PBCacheParam) ProtoMessage() {} func (*PBCacheParam) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{5} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{5} } func (m *PBCacheParam) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBCacheParam.Unmarshal(m, b) @@ -593,6 +622,41 @@ func (m *PBCacheParam) GetOverwrite() int32 { return 0 } +func (m *PBCacheParam) GetFilemode() uint32 { + if m != nil && m.Filemode != nil { + return *m.Filemode + } + return 0 +} + +func (m *PBCacheParam) GetLinktarget() []byte { + if m != nil { + return m.Linktarget + } + return nil +} + +func (m *PBCacheParam) GetModifytime() int64 { + if m != nil && m.Modifytime != nil { + return *m.Modifytime + } + return 0 +} + +func (m *PBCacheParam) GetAccesstime() int64 { + if m != nil && m.Accesstime != nil { + return *m.Accesstime + } + return 0 +} + +func (m *PBCacheParam) GetCreatetime() int64 { + if m != nil && m.Createtime != nil { + return *m.Createtime + } + return 0 +} + type PBCacheResult struct { Status *PBCacheStatus `protobuf:"varint,1,req,name=status,enum=protocol.PBCacheStatus" json:"status,omitempty"` Reason []byte `protobuf:"bytes,2,req,name=reason" json:"reason,omitempty"` @@ -605,7 +669,7 @@ func (m *PBCacheResult) Reset() { *m = PBCacheResult{} } func (m *PBCacheResult) String() string { return proto.CompactTextString(m) } func (*PBCacheResult) ProtoMessage() {} func (*PBCacheResult) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{6} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{6} } func (m *PBCacheResult) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBCacheResult.Unmarshal(m, b) @@ -656,7 +720,7 @@ func (m *PBHead) Reset() { *m = PBHead{} } func (m *PBHead) String() string { return proto.CompactTextString(m) } func (*PBHead) ProtoMessage() {} func (*PBHead) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{7} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{7} } func (m *PBHead) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBHead.Unmarshal(m, b) @@ -736,7 +800,7 @@ func (m *PBBodyDispatchTaskReq) Reset() { *m = PBBodyDispatchTaskReq{} } func (m *PBBodyDispatchTaskReq) String() string { return proto.CompactTextString(m) } func (*PBBodyDispatchTaskReq) ProtoMessage() {} func (*PBBodyDispatchTaskReq) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{8} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{8} } func (m *PBBodyDispatchTaskReq) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBBodyDispatchTaskReq.Unmarshal(m, b) @@ -774,7 +838,7 @@ func (m *PBBodyDispatchTaskRsp) Reset() { *m = PBBodyDispatchTaskRsp{} } func (m *PBBodyDispatchTaskRsp) String() string { return proto.CompactTextString(m) } func (*PBBodyDispatchTaskRsp) ProtoMessage() {} func (*PBBodyDispatchTaskRsp) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{9} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{9} } func (m *PBBodyDispatchTaskRsp) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBBodyDispatchTaskRsp.Unmarshal(m, b) @@ -812,7 +876,7 @@ func (m *PBBodySyncTimeRsp) Reset() { *m = PBBodySyncTimeRsp{} } func (m *PBBodySyncTimeRsp) String() string { return proto.CompactTextString(m) } func (*PBBodySyncTimeRsp) ProtoMessage() {} func (*PBBodySyncTimeRsp) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{10} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{10} } func (m *PBBodySyncTimeRsp) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBBodySyncTimeRsp.Unmarshal(m, b) @@ -850,7 +914,7 @@ func (m *PBBodySendFileReq) Reset() { *m = PBBodySendFileReq{} } func (m *PBBodySendFileReq) String() string { return proto.CompactTextString(m) } func (*PBBodySendFileReq) ProtoMessage() {} func (*PBBodySendFileReq) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{11} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{11} } func (m *PBBodySendFileReq) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBBodySendFileReq.Unmarshal(m, b) @@ -888,7 +952,7 @@ func (m *PBBodySendFileRsp) Reset() { *m = PBBodySendFileRsp{} } func (m *PBBodySendFileRsp) String() string { return proto.CompactTextString(m) } func (*PBBodySendFileRsp) ProtoMessage() {} func (*PBBodySendFileRsp) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{12} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{12} } func (m *PBBodySendFileRsp) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBBodySendFileRsp.Unmarshal(m, b) @@ -926,7 +990,7 @@ func (m *PBBodyCheckCacheReq) Reset() { *m = PBBodyCheckCacheReq{} } func (m *PBBodyCheckCacheReq) String() string { return proto.CompactTextString(m) } func (*PBBodyCheckCacheReq) ProtoMessage() {} func (*PBBodyCheckCacheReq) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{13} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{13} } func (m *PBBodyCheckCacheReq) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBBodyCheckCacheReq.Unmarshal(m, b) @@ -964,7 +1028,7 @@ func (m *PBBodyCheckCacheRsp) Reset() { *m = PBBodyCheckCacheRsp{} } func (m *PBBodyCheckCacheRsp) String() string { return proto.CompactTextString(m) } func (*PBBodyCheckCacheRsp) ProtoMessage() {} func (*PBBodyCheckCacheRsp) Descriptor() ([]byte, []int) { - return fileDescriptor_task_worker_4d87a065e00f024a, []int{14} + return fileDescriptor_task_worker_6fa3dbe88e12366c, []int{14} } func (m *PBBodyCheckCacheRsp) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PBBodyCheckCacheRsp.Unmarshal(m, b) @@ -1012,65 +1076,68 @@ func init() { proto.RegisterEnum("protocol.PBCmdType", PBCmdType_name, PBCmdType_value) } -func init() { proto.RegisterFile("task_worker.proto", fileDescriptor_task_worker_4d87a065e00f024a) } - -var fileDescriptor_task_worker_4d87a065e00f024a = []byte{ - // 906 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x54, 0xdb, 0x72, 0xe3, 0x44, - 0x10, 0x8d, 0x2c, 0x5f, 0xdb, 0x97, 0xc8, 0x93, 0x0b, 0x2a, 0x9e, 0x84, 0xd8, 0x62, 0x45, 0xa0, - 0xf2, 0xe0, 0x82, 0x17, 0x6a, 0xb7, 0xa8, 0x58, 0x76, 0xb0, 0x2b, 0x59, 0x59, 0x58, 0xce, 0x6e, - 0xb1, 0x2f, 0xd4, 0x44, 0x9a, 0x24, 0x2a, 0xeb, 0x96, 0x99, 0x71, 0x16, 0xf3, 0x13, 0x14, 0xaf, - 0xfc, 0x03, 0x5f, 0xc0, 0x67, 0xf0, 0x43, 0xd4, 0x8c, 0x64, 0x3b, 0x76, 0x12, 0x9e, 0xec, 0xe9, - 0xe9, 0xd3, 0xd3, 0xe7, 0xf4, 0x69, 0x41, 0x97, 0x63, 0x36, 0xff, 0xf5, 0x53, 0x4a, 0xe7, 0x84, - 0x9e, 0x66, 0x34, 0xe5, 0x29, 0xaa, 0xcb, 0x1f, 0x3f, 0x8d, 0xcc, 0x7f, 0x15, 0x00, 0xb7, 0x7f, - 0x1e, 0x46, 0x64, 0x40, 0x98, 0x8f, 0x34, 0xa8, 0xdf, 0x2c, 0xa2, 0x28, 0xc3, 0xfc, 0x4e, 0x57, - 0x8c, 0x92, 0xd5, 0x40, 0x2d, 0x28, 0xb3, 0xf0, 0x77, 0xa2, 0x97, 0x8c, 0x92, 0xa5, 0xa2, 0x26, - 0xa8, 0x71, 0xf0, 0xbd, 0xae, 0xca, 0xab, 0x53, 0x68, 0xf9, 0x69, 0x9c, 0x51, 0xc2, 0x18, 0x5f, - 0x66, 0x44, 0x2f, 0x1b, 0x25, 0xab, 0xd3, 0xd3, 0x4f, 0x57, 0xc5, 0x4f, 0xdd, 0xbe, 0x5d, 0xdc, - 0xcf, 0x96, 0x19, 0x41, 0xc7, 0xd0, 0x59, 0xe5, 0x93, 0x40, 0x16, 0xad, 0xc8, 0xa2, 0x1d, 0xa8, - 0x5e, 0x2f, 0x6e, 0x6e, 0x08, 0xd5, 0xab, 0x86, 0x62, 0xb5, 0xd0, 0xe7, 0x80, 0x38, 0xa6, 0xb7, - 0x84, 0x53, 0x12, 0x61, 0x1e, 0x3e, 0x10, 0xd9, 0x4e, 0xcd, 0x50, 0xac, 0x86, 0x6c, 0x30, 0x8c, - 0x48, 0x9c, 0x06, 0x44, 0xaf, 0x1b, 0x8a, 0xd5, 0x46, 0x08, 0x20, 0x0a, 0x93, 0x79, 0x8e, 0xd0, - 0x1b, 0xa2, 0x82, 0xf9, 0x0e, 0x5a, 0x39, 0xa9, 0x29, 0x61, 0x8b, 0x88, 0x3f, 0x43, 0x6b, 0x1f, - 0x6a, 0x94, 0x70, 0x5f, 0x94, 0x11, 0xcc, 0x2a, 0x2f, 0x3c, 0xaa, 0x8a, 0x47, 0xcd, 0xbf, 0x14, - 0x68, 0x48, 0x2e, 0x31, 0x4e, 0x02, 0x01, 0x15, 0x62, 0x06, 0x21, 0xdd, 0xd4, 0x22, 0xbf, 0xe5, - 0xf9, 0xa5, 0x47, 0x81, 0x04, 0xc7, 0xa4, 0x50, 0xaa, 0x03, 0xd5, 0x0c, 0x53, 0x1c, 0x33, 0xbd, - 0x6c, 0xa8, 0x56, 0x03, 0x59, 0x00, 0x61, 0x92, 0x2d, 0xb8, 0xa0, 0xc2, 0xf4, 0x8a, 0xa1, 0x5a, - 0xcd, 0xde, 0xe1, 0x63, 0xdd, 0xd6, 0x03, 0x39, 0x80, 0x26, 0x95, 0x1c, 0xf2, 0xd4, 0xaa, 0x84, - 0x37, 0x41, 0x25, 0xc9, 0x83, 0x5e, 0x33, 0x54, 0xab, 0x65, 0x5a, 0xd0, 0x74, 0xfb, 0x1e, 0xc7, - 0x7c, 0x98, 0x70, 0xba, 0x14, 0x77, 0x73, 0xb2, 0xdc, 0x0c, 0x8f, 0x87, 0x71, 0x31, 0x3c, 0xf3, - 0x1f, 0x05, 0xea, 0x6e, 0xbf, 0x90, 0xc4, 0x00, 0xd5, 0x8f, 0x03, 0x99, 0xd7, 0xec, 0x1d, 0xec, - 0xcc, 0x6c, 0xc5, 0x73, 0x5b, 0xa2, 0x23, 0x68, 0xa7, 0x0b, 0x9e, 0x2d, 0x78, 0x4c, 0x18, 0xc3, - 0xb7, 0x2b, 0x72, 0x87, 0xd0, 0x22, 0x94, 0xa6, 0x74, 0x15, 0x2d, 0xcb, 0xe8, 0xd7, 0xdb, 0x8d, - 0xff, 0x1f, 0xc7, 0x57, 0x50, 0x61, 0x1c, 0xf3, 0x9c, 0x5d, 0xb3, 0x77, 0xf4, 0x38, 0x69, 0x4d, - 0xcc, 0xfc, 0x28, 0x66, 0x6a, 0x63, 0xff, 0x8e, 0xb8, 0x42, 0x4a, 0xc1, 0x4d, 0x2a, 0x2c, 0x18, - 0xb4, 0x56, 0xc6, 0x2c, 0xc9, 0xc3, 0xca, 0xb3, 0x62, 0x7a, 0xd2, 0x5e, 0x85, 0x39, 0xca, 0xf2, - 0xb6, 0x0b, 0x8d, 0xf4, 0x81, 0xd0, 0x4f, 0x34, 0xe4, 0xc2, 0x81, 0x8a, 0x55, 0x31, 0x47, 0xd0, - 0x2e, 0x6a, 0x17, 0xea, 0xbc, 0x86, 0xaa, 0x68, 0x69, 0xc1, 0x64, 0xf9, 0x4e, 0xef, 0xb3, 0x2d, - 0x81, 0x44, 0xa2, 0x27, 0xaf, 0x45, 0x71, 0x4a, 0x30, 0x4b, 0x93, 0xfc, 0x69, 0xf3, 0x4f, 0x05, - 0xaa, 0x6e, 0x7f, 0x44, 0xb0, 0xd4, 0xef, 0x81, 0x50, 0x16, 0xa6, 0x49, 0x31, 0x8d, 0x36, 0x54, - 0x62, 0x7c, 0x1b, 0xfa, 0x1b, 0x97, 0x5c, 0xa7, 0xc1, 0x32, 0x22, 0x89, 0x14, 0xb2, 0x52, 0xec, - 0x81, 0x38, 0x97, 0xe5, 0x5e, 0xbc, 0x82, 0x9a, 0x1f, 0x07, 0x72, 0xb5, 0x2a, 0xb2, 0x8b, 0xed, - 0x31, 0xc5, 0x81, 0xdc, 0x2a, 0x0d, 0xea, 0xd7, 0x0b, 0x16, 0x26, 0x84, 0x31, 0xb9, 0x3f, 0x8d, - 0x9c, 0x30, 0x9b, 0x87, 0x41, 0xbe, 0x33, 0xe6, 0x0f, 0x70, 0xe4, 0xf6, 0xfb, 0x69, 0xb0, 0x1c, - 0x84, 0x2c, 0xc3, 0xdc, 0xbf, 0x9b, 0x61, 0x36, 0x9f, 0x92, 0x7b, 0xf4, 0x05, 0x94, 0xfd, 0x38, - 0x10, 0x1c, 0xd5, 0x17, 0x4c, 0x60, 0xbe, 0x79, 0x16, 0xcb, 0x32, 0xf4, 0xa5, 0x70, 0x87, 0xd0, - 0x6a, 0x05, 0x47, 0x8f, 0xe1, 0xb9, 0x8c, 0xe6, 0x37, 0xd0, 0xcd, 0xd1, 0xde, 0x32, 0xf1, 0x67, - 0x61, 0x4c, 0x04, 0xf2, 0x18, 0x3a, 0xc2, 0x94, 0x09, 0x4e, 0x52, 0x46, 0xfc, 0x34, 0xc9, 0x4d, - 0xa8, 0x9a, 0x6f, 0xd7, 0xc9, 0x24, 0x09, 0xf2, 0xe5, 0xbd, 0xdf, 0xd9, 0x14, 0xe5, 0x65, 0x17, - 0x99, 0x6f, 0x9e, 0xc0, 0x59, 0x86, 0x5e, 0xef, 0x76, 0x79, 0xbc, 0x8b, 0x2d, 0x3a, 0x7d, 0x0b, - 0x07, 0x39, 0xda, 0xbe, 0x23, 0xfe, 0xbc, 0xb0, 0xc2, 0x3d, 0xfa, 0x6a, 0xbd, 0xb8, 0xcf, 0xc0, - 0x37, 0x66, 0x34, 0x7f, 0x7c, 0x06, 0xce, 0x32, 0x64, 0xed, 0x3e, 0xff, 0xd4, 0x47, 0xf9, 0xfb, - 0x27, 0xdf, 0x42, 0x67, 0xe7, 0x6b, 0x59, 0x87, 0xb2, 0x33, 0x71, 0x86, 0xda, 0x1e, 0xaa, 0x81, - 0x7a, 0xf9, 0x71, 0xa2, 0x29, 0xf9, 0x9f, 0xef, 0xb4, 0xd2, 0xc9, 0xfb, 0xb5, 0x5f, 0x0b, 0x1b, - 0x36, 0xa1, 0xe6, 0x4c, 0xce, 0x27, 0x57, 0xce, 0x40, 0xdb, 0x13, 0x07, 0xef, 0xca, 0xb6, 0x87, - 0x9e, 0xa7, 0x29, 0xe8, 0x08, 0xba, 0xc3, 0xe9, 0x74, 0x32, 0xfd, 0x30, 0x1a, 0x5f, 0x0e, 0xcf, - 0xc7, 0xce, 0x60, 0xec, 0xfc, 0xa4, 0x95, 0xd0, 0x21, 0x68, 0x9b, 0xb0, 0x77, 0xf6, 0x5e, 0x44, - 0xd5, 0x93, 0xbf, 0xf3, 0x0f, 0x5d, 0xe1, 0xac, 0x03, 0xd8, 0x1f, 0x8c, 0x3d, 0xf7, 0x6c, 0x66, - 0x8f, 0x66, 0x67, 0xde, 0xc5, 0x74, 0xf8, 0xb3, 0xb6, 0xf7, 0x24, 0xe8, 0xb9, 0x9a, 0x82, 0xf6, - 0xa1, 0xe9, 0xfd, 0xe2, 0xd8, 0xb3, 0xf1, 0xbb, 0xa1, 0xc8, 0x2a, 0x6d, 0x05, 0x3c, 0x57, 0x53, - 0x65, 0x60, 0xe8, 0x0c, 0xce, 0xc7, 0x97, 0x32, 0xa3, 0xbc, 0x15, 0xf0, 0x5c, 0xad, 0x82, 0xba, - 0xd0, 0xb6, 0x47, 0x43, 0xfb, 0xc2, 0x3e, 0xb3, 0x47, 0x32, 0xa7, 0xba, 0x13, 0xf2, 0x5c, 0xad, - 0x86, 0x5a, 0x50, 0xbb, 0x72, 0x2e, 0x9c, 0xc9, 0x07, 0x47, 0xfb, 0xc3, 0xf9, 0x2f, 0x00, 0x00, - 0xff, 0xff, 0x77, 0xed, 0x91, 0x5e, 0xdb, 0x06, 0x00, 0x00, +func init() { proto.RegisterFile("task_worker.proto", fileDescriptor_task_worker_6fa3dbe88e12366c) } + +var fileDescriptor_task_worker_6fa3dbe88e12366c = []byte{ + // 957 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x54, 0xdd, 0x72, 0xe3, 0x34, + 0x18, 0xad, 0xed, 0xfc, 0x7e, 0x4e, 0x53, 0x57, 0xfd, 0xc1, 0xc3, 0x95, 0x31, 0x3b, 0xac, 0x29, + 0x4c, 0x2f, 0x3a, 0x70, 0xc3, 0xec, 0x0e, 0xd3, 0x38, 0x29, 0xc9, 0xb4, 0xeb, 0x98, 0x38, 0xdd, + 0x1d, 0xb8, 0x61, 0x54, 0x5b, 0x6d, 0x3d, 0x89, 0x7f, 0x2a, 0x29, 0x5d, 0xc2, 0x4b, 0xec, 0x70, + 0xcb, 0x3b, 0xf0, 0x04, 0xf0, 0x70, 0x8c, 0x64, 0x27, 0x69, 0xd2, 0x94, 0xab, 0x44, 0x47, 0xdf, + 0x91, 0xbf, 0x73, 0xbe, 0x23, 0xc1, 0x3e, 0xc7, 0x6c, 0xf2, 0xdb, 0xc7, 0x8c, 0x4e, 0x08, 0x3d, + 0xcd, 0x69, 0xc6, 0x33, 0xd4, 0x90, 0x3f, 0x61, 0x36, 0xb5, 0x3f, 0xa9, 0x00, 0x7e, 0xe7, 0x22, + 0x9e, 0x92, 0x2e, 0x61, 0x21, 0x32, 0xa0, 0x71, 0x3b, 0x9b, 0x4e, 0x73, 0xcc, 0xef, 0x4d, 0xc5, + 0x52, 0x9d, 0x26, 0x6a, 0x41, 0x85, 0xc5, 0x7f, 0x10, 0x53, 0xb5, 0x54, 0x47, 0x43, 0x3a, 0x68, + 0x49, 0xf4, 0xbd, 0xa9, 0xc9, 0xad, 0x53, 0x68, 0x85, 0x59, 0x92, 0x53, 0xc2, 0x18, 0x9f, 0xe7, + 0xc4, 0xac, 0x58, 0xaa, 0xd3, 0x3e, 0x33, 0x4f, 0x17, 0x87, 0x9f, 0xfa, 0x1d, 0xb7, 0xdc, 0x1f, + 0xcf, 0x73, 0x82, 0x8e, 0xa1, 0xbd, 0xa8, 0x27, 0x91, 0x3c, 0xb4, 0x2a, 0x0f, 0x6d, 0x43, 0xed, + 0x66, 0x76, 0x7b, 0x4b, 0xa8, 0x59, 0xb3, 0x14, 0xa7, 0x85, 0x3e, 0x07, 0xc4, 0x31, 0xbd, 0x23, + 0x9c, 0x92, 0x29, 0xe6, 0xf1, 0x23, 0x91, 0xed, 0xd4, 0x2d, 0xc5, 0x69, 0xca, 0x06, 0xe3, 0x29, + 0x49, 0xb2, 0x88, 0x98, 0x0d, 0x4b, 0x71, 0x76, 0x11, 0x02, 0x98, 0xc6, 0xe9, 0xa4, 0x60, 0x98, + 0x4d, 0x79, 0x02, 0x02, 0x48, 0xb2, 0x28, 0xbe, 0x9d, 0xf3, 0x38, 0x21, 0x26, 0x58, 0x8a, 0xa3, + 0x09, 0x0c, 0x87, 0xa1, 0xe8, 0x55, 0x60, 0xfa, 0x02, 0x0b, 0x29, 0xc1, 0x9c, 0x48, 0xac, 0x25, + 0x30, 0xfb, 0x1d, 0xb4, 0x0a, 0x43, 0x46, 0x84, 0xcd, 0xa6, 0x7c, 0x8b, 0x25, 0x7b, 0x50, 0xa7, + 0x84, 0x87, 0xa2, 0x05, 0xe1, 0x4a, 0xf5, 0x85, 0x86, 0x35, 0xd1, 0xb0, 0xfd, 0x97, 0x02, 0x4d, + 0xe9, 0x43, 0x82, 0xd3, 0x48, 0x50, 0xc5, 0x20, 0xa2, 0x98, 0xae, 0xce, 0x22, 0xbf, 0x17, 0xf5, + 0xea, 0x13, 0x20, 0xc5, 0x09, 0x29, 0x5d, 0x6e, 0x43, 0x2d, 0xc7, 0x14, 0x27, 0xcc, 0xac, 0x58, + 0x9a, 0xd3, 0x44, 0x0e, 0x40, 0x9c, 0xe6, 0x33, 0x2e, 0x6c, 0x60, 0x66, 0xd5, 0xd2, 0x1c, 0xfd, + 0xec, 0xf0, 0xa9, 0xe7, 0xcb, 0x61, 0x1e, 0x80, 0x4e, 0xa5, 0x86, 0xa2, 0xb4, 0x26, 0xe9, 0x3a, + 0x68, 0x24, 0x7d, 0x34, 0xeb, 0x96, 0xe6, 0xb4, 0x6c, 0x07, 0x74, 0xbf, 0x13, 0x70, 0xcc, 0x7b, + 0x29, 0xa7, 0x73, 0xb1, 0x37, 0x21, 0xf3, 0xd5, 0xe0, 0xa5, 0x2b, 0x72, 0xf0, 0xf6, 0x3f, 0x0a, + 0x34, 0xfc, 0x4e, 0x69, 0x89, 0x05, 0x5a, 0x98, 0x44, 0xb2, 0x4e, 0x3f, 0x3b, 0xd8, 0x98, 0xf7, + 0x42, 0xe7, 0xba, 0x45, 0x47, 0xb0, 0x9b, 0xcd, 0x78, 0x3e, 0xe3, 0x09, 0x61, 0x0c, 0xdf, 0x2d, + 0xc4, 0x1d, 0x42, 0x8b, 0x50, 0x9a, 0xd1, 0x05, 0x5a, 0x91, 0xe8, 0xd7, 0xeb, 0x8d, 0xff, 0x9f, + 0xc6, 0x57, 0x50, 0x65, 0x1c, 0xf3, 0x42, 0x9d, 0x7e, 0x76, 0xf4, 0xb4, 0x68, 0x29, 0xcc, 0xfe, + 0x57, 0x11, 0x43, 0x75, 0x71, 0x78, 0x4f, 0x7c, 0xe1, 0xa5, 0x10, 0x27, 0x2d, 0x16, 0x12, 0x5a, + 0x8b, 0x54, 0xab, 0x72, 0xb1, 0x08, 0xbc, 0x26, 0x13, 0xd2, 0x86, 0x5a, 0x99, 0xac, 0x8a, 0xdc, + 0xdd, 0x87, 0x66, 0xf6, 0x48, 0xe8, 0x47, 0x1a, 0x73, 0x11, 0x5f, 0xc5, 0xa9, 0xae, 0x45, 0xb2, + 0xb6, 0x25, 0x92, 0xf5, 0x2d, 0x91, 0x6c, 0x6c, 0x89, 0x64, 0x73, 0x4b, 0x24, 0x65, 0x74, 0xed, + 0x3e, 0xec, 0x96, 0xdd, 0x97, 0x03, 0x78, 0x0d, 0x35, 0xa1, 0x7a, 0xc6, 0xa4, 0x80, 0xf6, 0xd9, + 0x67, 0x6b, 0x33, 0x10, 0x85, 0x81, 0xdc, 0x16, 0xed, 0x53, 0x82, 0x59, 0x96, 0x16, 0xe2, 0xec, + 0x3f, 0x15, 0xa8, 0xf9, 0x9d, 0x3e, 0xc1, 0x72, 0x44, 0x8f, 0x84, 0xb2, 0x38, 0x4b, 0xcb, 0x81, + 0xef, 0x42, 0x35, 0xc1, 0x77, 0x71, 0xb8, 0x0a, 0xe2, 0x4d, 0x16, 0xcd, 0xa7, 0x24, 0x95, 0xb3, + 0xaa, 0x96, 0xd7, 0x54, 0xac, 0x2b, 0xf2, 0xda, 0xbe, 0x82, 0x7a, 0x98, 0x44, 0xf2, 0xe6, 0x57, + 0x65, 0x17, 0xeb, 0x49, 0x48, 0x22, 0x79, 0xe9, 0x0d, 0x68, 0xdc, 0xcc, 0x58, 0x9c, 0x12, 0xc6, + 0xa4, 0x3b, 0xcd, 0xc2, 0x52, 0x36, 0x89, 0xa3, 0xe2, 0x4a, 0xdb, 0x3f, 0xc0, 0x91, 0xdf, 0xe9, + 0x64, 0xd1, 0xbc, 0x1b, 0xb3, 0x1c, 0xf3, 0xf0, 0x7e, 0x8c, 0xd9, 0x64, 0x44, 0x1e, 0xd0, 0x17, + 0x50, 0x09, 0x93, 0x48, 0x68, 0xd4, 0x5e, 0xc8, 0x99, 0xfd, 0x66, 0x2b, 0x97, 0xe5, 0xe8, 0x4b, + 0x11, 0x40, 0xe1, 0xd5, 0x82, 0x8e, 0x9e, 0xd2, 0x0b, 0x1b, 0xed, 0x6f, 0x60, 0xbf, 0x60, 0x07, + 0xf3, 0x34, 0x1c, 0xc7, 0x09, 0x11, 0xcc, 0x63, 0x68, 0x0b, 0xeb, 0x53, 0x9c, 0x66, 0x8c, 0x84, + 0x59, 0x5a, 0xe4, 0x5c, 0xb3, 0xdf, 0x2e, 0x8b, 0x49, 0x1a, 0x15, 0xef, 0xc3, 0xc3, 0xc6, 0x65, + 0x54, 0x5e, 0x0e, 0xaa, 0xfd, 0xe6, 0x19, 0x9d, 0xe5, 0xe8, 0xf5, 0x66, 0x97, 0xc7, 0x9b, 0xdc, + 0xb2, 0xd3, 0xb7, 0x70, 0x50, 0xb0, 0xdd, 0x7b, 0x12, 0x4e, 0xca, 0x28, 0x3c, 0xa0, 0xaf, 0x96, + 0x6f, 0xc3, 0x16, 0xfa, 0x2a, 0xee, 0xf6, 0x8f, 0x5b, 0xe8, 0x2c, 0x47, 0xce, 0xe6, 0xe7, 0x9f, + 0xe7, 0xa8, 0xf8, 0xfe, 0xc9, 0xb7, 0xd0, 0xde, 0x78, 0xcc, 0x1b, 0x50, 0xf1, 0x86, 0x5e, 0xcf, + 0xd8, 0x41, 0x75, 0xd0, 0xae, 0x7e, 0x1d, 0x1a, 0x4a, 0xf1, 0xe7, 0x3b, 0x43, 0x3d, 0x79, 0xbf, + 0xcc, 0x6b, 0x19, 0x43, 0x1d, 0xea, 0xde, 0xf0, 0x62, 0x78, 0xed, 0x75, 0x8d, 0x1d, 0xb1, 0x08, + 0xae, 0x5d, 0xb7, 0x17, 0x04, 0x86, 0x82, 0x8e, 0x60, 0xbf, 0x37, 0x1a, 0x0d, 0x47, 0x1f, 0xfa, + 0x83, 0xab, 0xde, 0xc5, 0xc0, 0xeb, 0x0e, 0xbc, 0x9f, 0x0c, 0x15, 0x1d, 0x82, 0xb1, 0x82, 0x83, + 0xf3, 0xf7, 0x02, 0xd5, 0x4e, 0xfe, 0x2e, 0xde, 0xd2, 0x32, 0x59, 0x07, 0xb0, 0xd7, 0x1d, 0x04, + 0xfe, 0xf9, 0xd8, 0xed, 0x8f, 0xcf, 0x83, 0xcb, 0x51, 0xef, 0x67, 0x63, 0xe7, 0x19, 0x18, 0xf8, + 0x86, 0x82, 0xf6, 0x40, 0x0f, 0x7e, 0xf1, 0xdc, 0xf1, 0xe0, 0x5d, 0x4f, 0x54, 0xa9, 0x6b, 0x40, + 0xe0, 0x1b, 0x9a, 0x04, 0x7a, 0x5e, 0xf7, 0x62, 0x70, 0x25, 0x2b, 0x2a, 0x6b, 0x40, 0xe0, 0x1b, + 0x55, 0xb4, 0x0f, 0xbb, 0x6e, 0xbf, 0xe7, 0x5e, 0xba, 0xe7, 0x6e, 0x5f, 0xd6, 0xd4, 0x36, 0xa0, + 0xc0, 0x37, 0xea, 0xa8, 0x05, 0xf5, 0x6b, 0xef, 0xd2, 0x1b, 0x7e, 0xf0, 0x8c, 0x4f, 0xde, 0x7f, + 0x01, 0x00, 0x00, 0xff, 0xff, 0x37, 0x25, 0x6b, 0x57, 0x7a, 0x07, 0x00, 0x00, } diff --git a/src/backend/booster/bk_dist/common/protocol/task_worker.proto b/src/backend/booster/bk_dist/common/protocol/task_worker.proto index d8da0587603..efbd4eae27b 100644 --- a/src/backend/booster/bk_dist/common/protocol/task_worker.proto +++ b/src/backend/booster/bk_dist/common/protocol/task_worker.proto @@ -17,6 +17,9 @@ message PBFileDesc { optional string targetrelativepath = 7; optional uint32 filemode = 8; optional bytes linktarget = 9; + optional int64 modifytime = 10; + optional int64 accesstime = 11; + optional int64 createtime = 12; } message PBFileResult { @@ -55,6 +58,11 @@ message PBCacheParam { optional int64 size = 3; required bytes target = 4; optional int32 overwrite = 5; + optional uint32 filemode = 6; + optional bytes linktarget = 7; + optional int64 modifytime = 8; + optional int64 accesstime = 9; + optional int64 createtime = 10; } enum PBCacheStatus { diff --git a/src/backend/booster/bk_dist/common/pump/pump.go b/src/backend/booster/bk_dist/common/pump/pump.go index bf4560413e9..ee0ae70780f 100644 --- a/src/backend/booster/bk_dist/common/pump/pump.go +++ b/src/backend/booster/bk_dist/common/pump/pump.go @@ -150,6 +150,7 @@ func IsPump(env *env.Sandbox) bool { } func SupportPump(env *env.Sandbox) bool { + // return IsPump(env) && (runtime.GOOS == "windows" || runtime.GOOS == "darwin") return IsPump(env) && runtime.GOOS == "windows" } @@ -174,3 +175,17 @@ func PumpCacheSizeMaxMB(env *env.Sandbox) int32 { return -1 } + +func PumpMinActionNum(env *env.Sandbox) int32 { + strsize := env.GetEnv(dcEnv.KeyExecutorPumpMinActionNum) + if strsize != "" { + size, err := strconv.Atoi(strsize) + if err != nil { + return 0 + } else { + return int32(size) + } + } + + return 0 +} diff --git a/src/backend/booster/bk_dist/common/types/booster.go b/src/backend/booster/bk_dist/common/types/booster.go index 8e29bc7a8de..84e440a4fa8 100644 --- a/src/backend/booster/bk_dist/common/types/booster.go +++ b/src/backend/booster/bk_dist/common/types/booster.go @@ -109,6 +109,8 @@ type BoosterWorks struct { PumpCacheDir string PumpCacheSizeMaxMB int32 PumpCacheRemoveAll bool + PumpBlackList []string + PumpMinActionNum int32 ForceLocalList []string diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go index e00c68d7657..5c3b676d9dd 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go @@ -127,7 +127,7 @@ func (e *executor) skipLocalRetry() bool { } func (e *executor) executePreTask() (*dcSDK.BKDistCommand, error) { - blog.Infof("executor: try to execute pre-task from pid(%d)", e.req.Pid) + // blog.Infof("executor: try to execute pre-task from pid(%d)", e.req.Pid) defer e.mgr.work.Basic().UpdateJobStats(e.stats) dcSDK.StatsTimeNow(&e.stats.PreWorkEnterTime) @@ -155,8 +155,8 @@ func (e *executor) executePreTask() (*dcSDK.BKDistCommand, error) { } e.stats.PreWorkSuccess = true - delta := e.req.Stats.PreWorkEndTime.Time().Sub(e.req.Stats.PreWorkStartTime.Time()) - blog.Infof("executor: success to execute pre-task from pid(%d) within %s", e.req.Pid, delta.String()) + // delta := e.req.Stats.PreWorkEndTime.Time().Sub(e.req.Stats.PreWorkStartTime.Time()) + // blog.Infof("executor: success to execute pre-task from pid(%d) within %s", e.req.Pid, delta.String()) blog.Debugf("executor: success to execute pre-task from pid(%d) and got data: %v", e.req.Pid, r) return r, nil } diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go index 029b1cd9772..64385a2cd5e 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go @@ -135,8 +135,10 @@ func (m *Mgr) ExecuteTask( } // 没有申请到资源(或资源已释放) || 申请到资源但都失效了 - if !m.work.Resource().HasAvailableWorkers() || - m.work.Remote().TotalSlots() <= 0 { + // if !m.work.Resource().HasAvailableWorkers() || + // m.work.Remote().TotalSlots() <= 0 { + // !! 去掉申请到资源但失效的情况，因为该情况很可能是网络原因，再加资源也没有意义 + if !m.work.Resource().HasAvailableWorkers() { // check whether this task need remote worker, // apply resource when need, if not in appling, apply then if e.needRemoteResource() { diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go index 50a3b40aeee..f49c633abbb 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go @@ -43,9 +43,12 @@ func NewMgr(pCtx context.Context, work *types.Work) types.RemoteMgr { conf: work.Config(), resourceCheckTick: 5 * time.Second, sendCorkTick: 10 * time.Millisecond, - corkSize: 1024 * 512, // 512KB, it will delay much if too big - corkFiles: make(map[string]*[]*corkFile, 0), - memSlot: newMemorySlot(0), + // corkSize: 1024 * 512, // 512KB, it will delay much if too big + corkSize: 1024 * 10, + corkMaxSize: 1024 * 1024 * 100, + corkFiles: make(map[string]*[]*corkFile, 0), + memSlot: newMemorySlot(0), + largeFileSize: 1024 * 1024 * 100, // 100MB } } @@ -87,7 +90,10 @@ type Mgr struct { sendCorkChan chan bool corkMutex sync.RWMutex corkSize int64 + corkMaxSize int64 corkFiles map[string]*[]*corkFile + + largeFileSize int64 } type fileSendMap struct { @@ -131,6 +137,63 @@ func (fsm *fileSendMap) matchOrInsert(desc dcSDK.FileDesc) (*types.FileInfo, boo return info, false } +func (fsm *fileSendMap) matchOrInserts(descs []*dcSDK.FileDesc) []matchResult { + fsm.Lock() + defer fsm.Unlock() + + if fsm.cache == nil { + fsm.cache = make(map[string]*[]*types.FileInfo) + } + + result := make([]matchResult, 0, len(descs)) + for _, desc := range descs { + info := &types.FileInfo{ + FullPath: desc.FilePath, + Size: desc.FileSize, + LastModifyTime: desc.Lastmodifytime, + Md5: desc.Md5, + TargetRelativePath: desc.Targetrelativepath, + FileMode: desc.Filemode, + LinkTarget: desc.LinkTarget, + SendStatus: types.FileSending, + } + + c, ok := fsm.cache[desc.FilePath] + if !ok || c == nil || len(*c) == 0 { + infoList := []*types.FileInfo{info} + fsm.cache[desc.FilePath] = &infoList + result = append(result, matchResult{ + info: info, + match: false, + }) + continue + } + + matched := false + for _, ci := range *c { + if ci.Match(*desc) { + result = append(result, matchResult{ + info: ci, + match: true, + }) + matched = true + break + } + } + if matched { + continue + } + + *c = append(*c, info) + result = append(result, matchResult{ + info: info, + match: false, + }) + } + + return result +} + func (fsm *fileSendMap) updateStatus(desc dcSDK.FileDesc, status types.FileSendStatus) { fsm.Lock() defer fsm.Unlock() @@ -317,7 +380,7 @@ func (m *Mgr) resourceCheck(ctx context.Context) { // notify resource release m.work.Resource().Release(nil) // send and reset stat data - m.work.Resource().SendAndResetStats(false, 0) + m.work.Resource().SendAndResetStats(false, []int64{0}) // 重置最近一次使用时间 m.setLastUsed(0) @@ -347,7 +410,12 @@ func (m *Mgr) ExecuteTask(req *types.RemoteTaskExecuteRequest) (*types.RemoteTas defer dcSDK.StatsTimeNow(&req.Stats.RemoteWorkLeaveTime) m.work.Basic().UpdateJobStats(req.Stats) - req.Server = m.lockSlots(dcSDK.JobUsageRemoteExe) + // 如果有超过100MB的大文件，则在选择host时，作为选择条件 + fpath, _ := getMaxSizeFile(req, m.largeFileSize) + req.Server = m.lockSlots(dcSDK.JobUsageRemoteExe, fpath) + blog.Infof("remote: selected host(%s) with large file(%s)", + req.Server.Server, fpath) + dcSDK.StatsTimeNow(&req.Stats.RemoteWorkLockTime) defer dcSDK.StatsTimeNow(&req.Stats.RemoteWorkUnlockTime) defer m.unlockSlots(dcSDK.JobUsageRemoteExe, req.Server) @@ -496,11 +564,16 @@ func (m *Mgr) ensureFiles( len(fileDetails), m.work.ID(), pid, sandbox.Dir, fileDetails) rules := settings.FilterRules + // TODO : pump模式下，一次编译依赖的可能有上千个文件，现在的流程会随机的添加到cork发送队列 + // 需要保证一次编译的依赖同时插入到cork发送队列，这样可以尽快的启动远程编译，避免远程编译等待太久 var err error wg := make(chan error, len(fileDetails)+1) count := 0 r := make([]string, 0, 10) cleaner := make([]dcSDK.FileDesc, 0, 10) + corkFiles := make(map[string]*[]*corkFile, 0) + allServerCorkFiles := make(map[string]*[]*corkFile, 0) + filesNum := len(fileDetails) for _, fd := range fileDetails { // 修改远程目录 f := fd.File @@ -544,33 +617,163 @@ func (m *Mgr) ensureFiles( continue } count++ - go func(err chan<- error, host *dcProtocol.Host, req *dcSDK.BKDistFileSender) { - t := time.Now().Local() - err <- m.ensureSingleFile(handler, host, req, sandbox) - d := time.Now().Local().Sub(t) - if d > 200*time.Millisecond { - blog.Debugf("remote: single file cost time for work(%s) from pid(%d) to server(%s): %s, %s", - m.work.ID(), pid, host.Server, d.String(), req.Files[0].FilePath) + if !m.conf.SendCork { + go func(err chan<- error, host *dcProtocol.Host, req *dcSDK.BKDistFileSender) { + t := time.Now().Local() + err <- m.ensureSingleFile(handler, host, req, sandbox) + d := time.Now().Local().Sub(t) + if d > 200*time.Millisecond { + blog.Debugf("remote: single file cost time for work(%s) from pid(%d) to server(%s): %s, %s", + m.work.ID(), pid, host.Server, d.String(), req.Files[0].FilePath) + } + }(wg, s, sender) + } else { + // TODO : for send cork + cf := &corkFile{ + handler: handler, + host: s, + sandbox: sandbox, + file: &f, + resultchan: nil, } - }(wg, s, sender) + l, ok := corkFiles[s.Server] + if !ok { + // 预先分配好队列，避免频繁内存分配 + // newl := []*corkFile{cf} + newl := make([]*corkFile, 0, filesNum) + newl = append(newl, cf) + corkFiles[s.Server] = &newl + } else { + *l = append(*l, cf) + } + } } // 分发额外的内容 for _, s := range servers { - go func(host *dcProtocol.Host, req *dcSDK.BKDistFileSender) { - t := time.Now().Local() - _ = m.ensureSingleFile(handler, host, req, sandbox) - d := time.Now().Local().Sub(t) - if d > 200*time.Millisecond { - blog.Debugf("remote: single file cost time for work(%s) from pid(%d) to server(%s): %s, %s", - m.work.ID(), pid, host.Server, d.String(), req.Files[0].FilePath) + if !m.conf.SendCork { + go func(host *dcProtocol.Host, req *dcSDK.BKDistFileSender) { + t := time.Now().Local() + _ = m.ensureSingleFile(handler, host, req, sandbox) + d := time.Now().Local().Sub(t) + if d > 200*time.Millisecond { + blog.Debugf("remote: single file cost time for work(%s) from pid(%d) to server(%s): %s, %s", + m.work.ID(), pid, host.Server, d.String(), req.Files[0].FilePath) + } + }(s, sender) + } else { + // TODO : for send cork + cf := &corkFile{ + handler: handler, + host: s, + sandbox: sandbox, + file: &f, + resultchan: nil, } - }(s, sender) + l, ok := allServerCorkFiles[s.Server] + if !ok { + // 预先分配好队列，避免频繁内存分配 + // newl := []*corkFile{cf} + newl := make([]*corkFile, 0, filesNum) + newl = append(newl, cf) + allServerCorkFiles[s.Server] = &newl + } else { + *l = append(*l, cf) + } + } + } + } + + if m.conf.SendCork { + blog.Debugf("remote: ready to ensure multi %d cork files for work(%s) from pid(%d) to server", + count, m.work.ID(), pid) + + for server, fs := range corkFiles { + totalFileNum := len(*fs) + descs := make([]*dcSDK.FileDesc, 0, totalFileNum) + for _, v := range *fs { + descs = append(descs, v.file) + } + results := m.checkOrLockCorkFiles(server, descs) + blog.Debugf("remote: got %d results for %d cork files count:%d for work(%s) from pid(%d) to server", + len(results), len(descs), count, m.work.ID(), pid) + needSendCorkFiles := make([]*corkFile, 0, totalFileNum) + for i, v := range results { + if v.match { + // 已发送完成的不启动协程了 + if v.info.SendStatus == types.FileSendSucceed { + wg <- nil + continue + } else if v.info.SendStatus == types.FileSendFailed { + wg <- types.ErrSendFileFailed + continue + } + } else { + // 不在缓存，意味着之前没有发送过 + (*fs)[i].resultchan = make(chan corkFileResult, 1) + needSendCorkFiles = append(needSendCorkFiles, (*fs)[i]) + } + + // 启动协程跟踪未发送完成的文件 + c := (*fs)[i] + go func(err chan<- error, c *corkFile, r matchResult) { + err <- m.ensureSingleCorkFile(c, r) + }(wg, c, v) + } + + // TODO : 检查是否在server端有缓存了，如果有，则无需发送，调用 checkBatchCache + + blog.Infof("total %d cork files, need send %d files", totalFileNum, len(needSendCorkFiles)) + // append to cork files queue + _ = m.appendCorkFiles(server, needSendCorkFiles) + + // notify send + m.sendCorkChan <- true + } + + // same with corkFiles, but do not notify wg + for server, fs := range allServerCorkFiles { + totalFileNum := len(*fs) + descs := make([]*dcSDK.FileDesc, 0, totalFileNum) + for _, v := range *fs { + descs = append(descs, v.file) + } + results := m.checkOrLockCorkFiles(server, descs) + needSendCorkFiles := make([]*corkFile, 0, totalFileNum) + for i, v := range results { + if v.match { + // 已发送完成的不启动协程了 + if v.info.SendStatus == types.FileSendSucceed { + continue + } else if v.info.SendStatus == types.FileSendFailed { + continue + } + } else { + // 不在缓存，意味着之前没有发送过 + (*fs)[i].resultchan = make(chan corkFileResult, 1) + needSendCorkFiles = append(needSendCorkFiles, (*fs)[i]) + } + + // 启动协程跟踪未发送完成的文件 + c := (*fs)[i] + go func(c *corkFile, r matchResult) { + _ = m.ensureSingleCorkFile(c, r) + }(c, v) + } + + blog.Infof("total %d cork files, need send %d files", totalFileNum, len(needSendCorkFiles)) + // append to cork files queue + _ = m.appendCorkFiles(server, needSendCorkFiles) + + // notify send + m.sendCorkChan <- true } } for i := 0; i < count; i++ { if err = <-wg; err != nil { + blog.Infof("remote: failed to ensure multi %d files for work(%s) from pid(%d) to server with err:%v", + count, m.work.ID(), pid, err) return nil, err } } @@ -632,18 +835,18 @@ func (m *Mgr) ensureSingleFile( return nil } - // send like tcp cork - if m.conf.SendCork { - retcode, err := m.sendFileWithCork(handler, &desc, host, sandbox) - if err != nil || retcode != 0 { - blog.Warnf("remote: execute send cork file(%s) for work(%s) to server(%s) failed: %v, retcode:%d", - desc.FilePath, m.work.ID(), host.Server, err, retcode) - } else { - blog.Debugf("remote: execute send cork file(%s) for work(%s) to server(%s) succeed", - desc.FilePath, m.work.ID(), host.Server) - return nil - } - } + // // send like tcp cork + // if m.conf.SendCork { + // retcode, err := m.sendFileWithCork(handler, &desc, host, sandbox) + // if err != nil || retcode != 0 { + // blog.Warnf("remote: execute send cork file(%s) for work(%s) to server(%s) failed: %v, retcode:%d", + // desc.FilePath, m.work.ID(), host.Server, err, retcode) + // } else { + // blog.Debugf("remote: execute send cork file(%s) for work(%s) to server(%s) succeed", + // desc.FilePath, m.work.ID(), host.Server) + // return nil + // } + // } blog.Debugf("remote: try to ensure single file(%s) for work(%s) to server(%s), going to send this file", desc.FilePath, m.work.ID(), host.Server) @@ -681,6 +884,67 @@ func (m *Mgr) ensureSingleFile( return nil } +// ensureSingleCorkFile 保证给到的第一个文件被正确分发到目标机器上, 若给到的文件多于一个, 多余的部分会被忽略 +func (m *Mgr) ensureSingleCorkFile(c *corkFile, r matchResult) (err error) { + status := r.info.SendStatus + host := c.host + desc := c.file + + blog.Debugf("remote: start ensure single cork file(%s) for work(%s) to server(%s)", + desc.FilePath, m.work.ID(), host.Server) + + // 已经有人发送了文件, 等待文件就绪 + if r.match { + blog.Debugf("remote: try to ensure single cork file(%s) for work(%s) to server(%s), "+ + "some one is sending this file", desc.FilePath, m.work.ID(), host.Server) + tick := time.NewTicker(m.checkSendFileTick) + defer tick.Stop() + + for status == types.FileSending { + select { + case <-tick.C: + status, _ = m.checkOrLockSendFile(host.Server, *desc) + } + } + + switch status { + case types.FileSendFailed: + blog.Errorf("remote: end ensure single cork file(%s) for work(%s) to server(%s), "+ + "file already sent and failed", desc.FilePath, m.work.ID(), host.Server) + return types.ErrSendFileFailed + case types.FileSendSucceed: + blog.Debugf("remote: end ensure single cork file(%s) for work(%s) to server(%s) succeed", + desc.FilePath, m.work.ID(), host.Server) + return nil + default: + blog.Errorf("remote: end ensure single cork file(%s) for work(%s) to server(%s), "+ + " with unknown status", desc.FilePath, m.work.ID(), host.Server) + return fmt.Errorf("unknown cork file send status: %s", status.String()) + } + } + + // send like tcp cork + blog.Debugf("remote: start wait result for send single cork file(%s) for work(%s) to server(%s)", + desc.FilePath, m.work.ID(), host.Server) + retcode, err := m.waitCorkFileResult(c) + // blog.Debugf("remote: end wait result for send single cork file(%s) for work(%s) to server(%s)", + // desc.FilePath, m.work.ID(), host.Server) + if err != nil { + blog.Warnf("remote: end ensure single cork file(%s) for work(%s) to server(%s) failed: %v, retcode:%d", + desc.FilePath, m.work.ID(), host.Server, err, retcode) + return err + } else if retcode != 0 { + blog.Warnf("remote: end ensure single cork file(%s) for work(%s) to server(%s) failed: %v, retcode:%d", + desc.FilePath, m.work.ID(), host.Server, err, retcode) + return fmt.Errorf("remote: send cork files(%s) for work(%s) to server(%s) failed, got retCode %d", + desc.FilePath, m.work.ID(), host.Server, retcode) + } else { + blog.Debugf("remote: end ensure single cork file(%s) for work(%s) to server(%s) succeed", + desc.FilePath, m.work.ID(), host.Server) + return nil + } +} + func (m *Mgr) checkSingleCache( handler dcSDK.RemoteWorkerHandler, host *dcProtocol.Host, @@ -744,7 +1008,7 @@ func (m *Mgr) checkOrLockSendFile(server string, desc dcSDK.FileDesc) (types.Fil t2 := time.Now().Local() if d1 := t2.Sub(t1); d1 > 50*time.Millisecond { - blog.Infof("check cache lock wait too long server(%s): %s", server, d1.String()) + blog.Debugf("check cache lock wait too long server(%s): %s", server, d1.String()) } defer func() { @@ -764,6 +1028,24 @@ func (m *Mgr) checkOrLockSendFile(server string, desc dcSDK.FileDesc) (types.Fil return info.SendStatus, match } +type matchResult struct { + info *types.FileInfo + match bool +} + +// checkOrLockCorkFiles 批量检查目标file的sendStatus, 如果已经被发送, 则返回当前状态和true; 如果没有被发送过, 则将其置于sending, 并返回false +func (m *Mgr) checkOrLockCorkFiles(server string, descs []*dcSDK.FileDesc) []matchResult { + m.fileSendMutex.Lock() + target, ok := m.fileSendMap[server] + if !ok { + target = &fileSendMap{} + m.fileSendMap[server] = target + } + m.fileSendMutex.Unlock() + + return target.matchOrInserts(descs) +} + func (m *Mgr) updateSendFile(server string, desc dcSDK.FileDesc, status types.FileSendStatus) { m.fileSendMutex.Lock() target, ok := m.fileSendMap[server] @@ -1062,8 +1344,8 @@ func (m *Mgr) getCachedToolChainStatus(server string, toolChainKey string) (type return types.FileSendUnknown, nil } -func (m *Mgr) lockSlots(usage dcSDK.JobUsage) *dcProtocol.Host { - return m.resource.Lock(usage) +func (m *Mgr) lockSlots(usage dcSDK.JobUsage, f string) *dcProtocol.Host { + return m.resource.Lock(usage, f) } func (m *Mgr) unlockSlots(usage dcSDK.JobUsage, host *dcProtocol.Host) { @@ -1270,7 +1552,37 @@ func (m *Mgr) sendFileWithCork(handler dcSDK.RemoteWorkerHandler, return msg.retcode, msg.err } -func (m *Mgr) getCorkFiles(getall bool) []*[]*corkFile { +func (m *Mgr) appendCorkFiles(server string, cfs []*corkFile) error { + // append to file queue + m.corkMutex.Lock() + if l, ok := m.corkFiles[server]; ok { + *l = append(*l, cfs...) + } else { + // 队列分配大点，避免频繁分配 + newlen := len(cfs) * 10 + newl := make([]*corkFile, 0, newlen) + newl = append(newl, cfs...) + m.corkFiles[server] = &newl + } + m.corkMutex.Unlock() + + // for _, v := range cfs { + // blog.Infof("remote: appended cork file[%s] ", v.file.FilePath) + // } + + // notify send + m.sendCorkChan <- true + + return nil +} + +// wait for result +func (m *Mgr) waitCorkFileResult(cf *corkFile) (int32, error) { + msg := <-cf.resultchan + return msg.retcode, msg.err +} + +func (m *Mgr) getCorkFiles(sendanyway bool) []*[]*corkFile { m.corkMutex.Lock() defer m.corkMutex.Unlock() @@ -1279,11 +1591,19 @@ func (m *Mgr) getCorkFiles(getall bool) []*[]*corkFile { // srcfiles := *v var totalsize int64 index := -1 - if getall { - index = len(*v) - 1 + if sendanyway { + // index = len(*v) - 1 + for index = range *v { + totalsize += (*v)[index].file.FileSize + // 如果数据包超过 m.corkMaxSize 了，则停止获取，下次再发 + if totalsize > m.corkMaxSize { + break + } + } } else { for index = range *v { totalsize += (*v)[index].file.FileSize + // 如果数据包超过 m.corkSize 了，则停止获取，下次再发 if totalsize > m.corkSize { break } @@ -1291,7 +1611,7 @@ func (m *Mgr) getCorkFiles(getall bool) []*[]*corkFile { } if index >= 0 { - if totalsize > m.corkSize || getall { + if sendanyway || (!sendanyway && totalsize > m.corkSize) { // get files num := index + 1 start := 0 @@ -1310,6 +1630,12 @@ func (m *Mgr) getCorkFiles(getall bool) []*[]*corkFile { } } + // for _, sv := range result { + // for _, v := range *sv { + // blog.Infof("remote: selected cork file[%s] ", v.file.FilePath) + // } + // } + return result } @@ -1352,6 +1678,7 @@ func (m *Mgr) sendFilesWithCorkSameHost(files []*corkFile) { var totalsize int64 req := &dcSDK.BKDistFileSender{} + req.Files = make([]dcSDK.FileDesc, 0, len(files)) for _, v := range files { req.Files = append(req.Files, *v.file) totalsize += v.file.FileSize @@ -1413,7 +1740,11 @@ func (m *Mgr) sendFilesWithCorkSameHost(files []*corkFile) { for _, v := range files { m.updateSendFile(host.Server, *v.file, status) + blog.Debugf("remote: ready ensure single cork file(%s) to status(%s) for work(%s) to server(%s)", + v.file.FilePath, status, m.work.ID(), host.Server) v.resultchan <- resultchan + // blog.Infof("remote: end send file[%s] with cork to server %s tick for work: %s with err:%v, retcode:%d", + // v.file.FilePath, host.Server, m.work.ID(), err, retcode) } blog.Infof("remote: end send %d files with cork to server %s tick for work: %s with err:%v, retcode:%d", diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/remote/slots.go b/src/backend/booster/bk_dist/controller/pkg/manager/remote/slots.go index 48dfa1be685..1bd3ad4248d 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/remote/slots.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/remote/slots.go @@ -22,9 +22,10 @@ import ( ) type lockWorkerMessage struct { - jobUsage dcSDK.JobUsage - toward *dcProtocol.Host - result chan *dcProtocol.Host + jobUsage dcSDK.JobUsage + toward *dcProtocol.Host + result chan *dcProtocol.Host + largeFile string } type lockWorkerChan chan lockWorkerMessage @@ -157,15 +158,16 @@ func (wr *resource) Handle(ctx context.Context) { } // Lock get an usage lock, success with true, failed with false -func (wr *resource) Lock(usage dcSDK.JobUsage) *dcProtocol.Host { +func (wr *resource) Lock(usage dcSDK.JobUsage, f string) *dcProtocol.Host { if !wr.handling { return nil } msg := lockWorkerMessage{ - jobUsage: usage, - toward: nil, - result: make(chan *dcProtocol.Host, 1), + jobUsage: usage, + toward: nil, + result: make(chan *dcProtocol.Host, 1), + largeFile: f, } // send a new lock request @@ -311,14 +313,63 @@ func (wr *resource) getWorkerWithMostFreeSlots() *worker { return w } -func (wr *resource) occupyWorkerSlots() *dcProtocol.Host { +// 大文件优先 +func (wr *resource) getWorkerLargeFileFirst(f string) *worker { + var w *worker + max := 0 + inlargequeue := false + for _, worker := range wr.worker { + if worker.disabled { + continue + } + + free := worker.totalSlots - worker.occupiedSlots + + // 在资源空闲时，大文件优先 + if free > worker.totalSlots/2 && worker.hasFile(f) { + // if free > 0 && worker.hasFile(f) { + if !inlargequeue { // first in large queue + inlargequeue = true + max = free + w = worker + } else { + if free >= max { + max = free + w = worker + } + } + continue + } + + if free >= max && !inlargequeue { + max = free + w = worker + } + } + if w == nil { + w = wr.worker[0] + } + + if f != "" && !w.hasFile(f) { + w.largefiles = append(w.largefiles, f) + } + + return w +} + +func (wr *resource) occupyWorkerSlots(f string) *dcProtocol.Host { wr.workerLock.Lock() defer wr.workerLock.Unlock() - worker := wr.getWorkerWithMostFreeSlots() - _ = worker.occupySlot() + var w *worker + if f == "" { + w = wr.getWorkerWithMostFreeSlots() + } else { + w = wr.getWorkerLargeFileFirst(f) + } + _ = w.occupySlot() - return worker.host + return w.host } func (wr *resource) freeWorkerSlots(host *dcProtocol.Host) { @@ -382,7 +433,7 @@ func (wr *resource) getSlot(msg lockWorkerMessage) { wr.occupiedSlots++ blog.Infof("remote slot: total slots:%d occupied slots:%d, remote slot available", wr.totalSlots, wr.occupiedSlots) - msg.result <- wr.occupyWorkerSlots() + msg.result <- wr.occupyWorkerSlots(msg.largeFile) satisfied = true } } @@ -390,8 +441,9 @@ func (wr *resource) getSlot(msg lockWorkerMessage) { if !satisfied { blog.Infof("remote slot: total slots:%d occupied slots:%d, remote slot not available", wr.totalSlots, wr.occupiedSlots) - wr.waitingList.PushBack(usage) - wr.waitingList.PushBack(msg.result) + // wr.waitingList.PushBack(usage) + // wr.waitingList.PushBack(msg.result) + wr.waitingList.PushBack(&msg) } } @@ -404,26 +456,30 @@ func (wr *resource) putSlot(msg lockWorkerMessage) { // check whether other waiting is satisfied now if wr.waitingList.Len() > 0 { - index := 0 + // index := 0 for e := wr.waitingList.Front(); e != nil; e = e.Next() { - if index%2 == 0 { - usage := e.Value.(dcSDK.JobUsage) - set := wr.getUsageSet(usage) - if wr.isIdle(set) { - set.occupied++ - wr.occupiedSlots++ + // if index%2 == 0 { + msg := e.Value.(*lockWorkerMessage) + // usage := e.Value.(dcSDK.JobUsage) + // set := wr.getUsageSet(usage) + set := wr.getUsageSet(msg.jobUsage) + if wr.isIdle(set) { + set.occupied++ + wr.occupiedSlots++ - chanElement := e.Next() - chanElement.Value.(chan *dcProtocol.Host) <- wr.occupyWorkerSlots() + msg.result <- wr.occupyWorkerSlots(msg.largeFile) - // delete this element - wr.waitingList.Remove(e) - wr.waitingList.Remove(chanElement) + // chanElement := e.Next() + // chanElement.Value.(chan *dcProtocol.Host) <- wr.occupyWorkerSlots() - break - } + // delete this element + wr.waitingList.Remove(e) + // wr.waitingList.Remove(chanElement) + + break } - index++ + // } + // index++ } } } @@ -435,6 +491,9 @@ type worker struct { host *dcProtocol.Host totalSlots int occupiedSlots int + // > 100MB + largefiles []string + largefiletotalsize uint64 } func (wr *worker) occupySlot() error { @@ -447,6 +506,18 @@ func (wr *worker) freeSlot() error { return nil } +func (wr *worker) hasFile(f string) bool { + if len(wr.largefiles) > 0 { + for _, v := range wr.largefiles { + if v == f { + return true + } + } + } + + return false +} + // by tming to limit local memory usage func newMemorySlot(maxSlots int64) *memorySlot { minmemroy := int64(2 * 1024 * 1024 * 1024) // 2GB @@ -454,9 +525,11 @@ func newMemorySlot(maxSlots int64) *memorySlot { if maxSlots <= 0 { v, err := mem.VirtualMemory() if err != nil { + blog.Infof("memory slot: failed to get virtaul memory with err:%v", err) maxSlots = int64((runtime.NumCPU() - 2)) * 1024 * 1024 * 1024 } else { - maxSlots = int64(v.Total) - minmemroy + // maxSlots = int64(v.Total) - minmemroy + maxSlots = int64(v.Total) / 2 } } diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go b/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go index a9b2be0d2fc..99a69a202c0 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go @@ -20,7 +20,7 @@ import ( ) func getFileDetailsFromExecuteRequest(req *types.RemoteTaskExecuteRequest) []*types.FilesDetails { - fd := make([]*types.FilesDetails, 0, 100) + fd := make([]*types.FilesDetails, 0, len(req.Req.Commands[0].Inputfiles)) for _, c := range req.Req.Commands { for _, f := range c.Inputfiles { fd = append(fd, &types.FilesDetails{ @@ -32,6 +32,25 @@ func getFileDetailsFromExecuteRequest(req *types.RemoteTaskExecuteRequest) []*ty return fd } +func getMaxSizeFile(req *types.RemoteTaskExecuteRequest, threshold int64) (string, int64) { + var maxsize int64 + fpath := "" + for _, c := range req.Req.Commands { + for _, v := range c.Inputfiles { + if v.FileSize > maxsize { + fpath = v.FilePath + maxsize = v.FileSize + } + } + } + + if maxsize > threshold { + return fpath, maxsize + } + + return "", 0 +} + // updateTaskRequestInputFilesReady 根据给定的baseDirs, 标记request中对应index的文件为"已经发送", 可以直接使用 func updateTaskRequestInputFilesReady(req *types.RemoteTaskExecuteRequest, baseDirs []string) error { index := 0 diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/resource/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/resource/mgr.go index a563d9b8771..c94f55c4911 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/resource/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/resource/mgr.go @@ -154,7 +154,10 @@ func (m *Mgr) GetNewlyTaskID() string { // SetSpecificHosts set the specific worker list instead of applying func (m *Mgr) SetSpecificHosts(hostList []string) { m.reslock.Lock() - defer m.reslock.Unlock() + defer func() { + m.reslock.Unlock() + m.onResChanged() + }() m.resources = append(m.resources, &Res{ taskid: "", status: ResourceSpecified, @@ -169,7 +172,7 @@ func (m *Mgr) SetSpecificHosts(hostList []string) { info.ResourceApplied() } - m.onResChanged() + // m.onResChanged() } // GetHosts return the worker list @@ -515,15 +518,19 @@ func (m *Mgr) SendStats(brief bool) error { return nil } -func (m *Mgr) SendAndResetStats(brief bool, t int64) error { +// send stats and reset after sent, if brief true, then will not send the job stats +// !! this will call m.work.Lock() , to avoid dead lock +func (m *Mgr) SendAndResetStats(brief bool, resapplytimes []int64) error { - data, _ := m.getSendStatsData(false, t) - go m.sendStatsData(data) + for _, t := range resapplytimes { + data, _ := m.getSendStatsData(brief, t) + go m.sendStatsData(data) - // reset stat - m.work.Lock() - m.work.Basic().ResetStat() - m.work.Unlock() + // reset stat + m.work.Lock() + m.work.Basic().ResetStat() + m.work.Unlock() + } return nil } @@ -682,7 +689,10 @@ func (m *Mgr) inspectInfo(taskID string) { s = ResourceApplyFailed } - m.clearOldInvalidRes(&info) + resapplytimes, err := m.clearOldInvalidRes(&info) + if err == nil { + m.SendAndResetStats(false, resapplytimes) + } m.addRes(&info, s) m.updateApplyEndStatus(s == ResourceApplySucceed) @@ -691,7 +701,10 @@ func (m *Mgr) inspectInfo(taskID string) { return case engine.TaskStatusFinish, engine.TaskStatusFailed: - m.clearOldInvalidRes(&info) + resapplytimes, err := m.clearOldInvalidRes(&info) + if err == nil { + m.SendAndResetStats(false, resapplytimes) + } m.addRes(&info, ResourceApplyFailed) m.updateApplyEndStatus(false) @@ -703,14 +716,14 @@ func (m *Mgr) inspectInfo(taskID string) { } // clean old resources which host list is empty, and notify server to terminate these resources -func (m *Mgr) clearOldInvalidRes(info *v2.RespTaskInfo) error { +func (m *Mgr) clearOldInvalidRes(info *v2.RespTaskInfo) ([]int64, error) { blog.Infof("resource: ready check and clean old invalid resource") m.reslock.Lock() defer m.reslock.Unlock() if len(m.resources) == 0 { - return nil + return nil, nil } needrelease := false @@ -725,9 +738,10 @@ func (m *Mgr) clearOldInvalidRes(info *v2.RespTaskInfo) error { } if !needrelease { - return nil + return nil, nil } + resapplytimes := []int64{} newres := []*Res{} for _, r := range m.resources { // do nothing with current task info, it mabye need by others @@ -738,18 +752,10 @@ func (m *Mgr) clearOldInvalidRes(info *v2.RespTaskInfo) error { if len(r.taskInfo.HostList) == 0 { m.releaseOne(nil, r) + // 确保 m.reslock.Lock() 和 m.work.Lock() 不要相互包含，避免死锁 // TODO : send detail stat data and reset stat data - - // // send stat - // data, _ := m.getSendStatsData(false, r.applyTime.UnixNano()) - // go m.sendStatsData(data) - - // // reset stat - // m.work.Lock() - // m.work.Basic().ResetStat() - // m.work.Unlock() - - m.SendAndResetStats(false, r.applyTime.UnixNano()) + // m.SendAndResetStats(false, r.applyTime.UnixNano()) + resapplytimes = append(resapplytimes, r.applyTime.UnixNano()) } else { newres = append(newres, r) @@ -757,7 +763,7 @@ func (m *Mgr) clearOldInvalidRes(info *v2.RespTaskInfo) error { } m.resources = newres - return nil + return resapplytimes, nil } func (m *Mgr) addRes(info *v2.RespTaskInfo, status Status) error { @@ -765,18 +771,6 @@ func (m *Mgr) addRes(info *v2.RespTaskInfo, status Status) error { return nil } - changed := false - m.reslock.Lock() - defer func() { - m.reslock.Unlock() - if changed { - m.onResChanged() - } - }() - - // send stat data with the newly taskid(resource id) - m.newlyTaskID = info.TaskID - // TODO : reset stat data to new status // 更新work info状态 m.work.Lock() @@ -800,6 +794,18 @@ func (m *Mgr) addRes(info *v2.RespTaskInfo, status Status) error { } m.work.Unlock() + changed := false + m.reslock.Lock() + defer func() { + m.reslock.Unlock() + if changed { + m.onResChanged() + } + }() + + // send stat data with the newly taskid(resource id) + m.newlyTaskID = info.TaskID + for _, r := range m.resources { if r.taskid == info.TaskID { // if resource in release status, do not change it diff --git a/src/backend/booster/bk_dist/controller/pkg/types/interface.go b/src/backend/booster/bk_dist/controller/pkg/types/interface.go index 5892440fcb4..e40cd3a3494 100644 --- a/src/backend/booster/bk_dist/controller/pkg/types/interface.go +++ b/src/backend/booster/bk_dist/controller/pkg/types/interface.go @@ -152,7 +152,8 @@ type ResourceMgr interface { SendStats(brief bool) error // send stats and reset after sent, if brief true, then will not send the job stats - SendAndResetStats(brief bool, t int64) error + // !! this will call m.work.Lock() , to avoid dead lock + SendAndResetStats(brief bool, resapplytimes []int64) error // get resource status GetStatus() *v2.RespTaskInfo diff --git a/src/backend/booster/bk_dist/handler/ue4/cc/error.go b/src/backend/booster/bk_dist/handler/ue4/cc/error.go index 257b23f7084..b97045ba05f 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cc/error.go +++ b/src/backend/booster/bk_dist/handler/ue4/cc/error.go @@ -36,4 +36,8 @@ var ( ErrorNotSupportConftest = fmt.Errorf("tmp.conftest. must be local") ErrorNotSupportOutputStdout = fmt.Errorf("output with - to stdout, must be local") ErrorNotSupportGch = fmt.Errorf("output with .gch, must be local") + ErrorNoPumpHeadFile = fmt.Errorf("pump head file not exist") + ErrorNoDependFile = fmt.Errorf("depend file not exist") + ErrorNotSupportRemote = fmt.Errorf("not support to remote execute") + ErrorInPumpBlack = fmt.Errorf("in pump black list") ) diff --git a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go index 72ac91e22a2..b3533aa5102 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go @@ -10,24 +10,33 @@ package cc import ( + // "bytes" "fmt" "io/ioutil" "os" "path/filepath" "runtime" + "strconv" "strings" + "time" + "github.com/Tencent/bk-ci/src/booster/bk_dist/common/env" dcEnv "github.com/Tencent/bk-ci/src/booster/bk_dist/common/env" dcFile "github.com/Tencent/bk-ci/src/booster/bk_dist/common/file" "github.com/Tencent/bk-ci/src/booster/bk_dist/common/protocol" + dcPump "github.com/Tencent/bk-ci/src/booster/bk_dist/common/pump" dcSDK "github.com/Tencent/bk-ci/src/booster/bk_dist/common/sdk" dcSyscall "github.com/Tencent/bk-ci/src/booster/bk_dist/common/syscall" + dcUtil "github.com/Tencent/bk-ci/src/booster/bk_dist/common/util" commonUtil "github.com/Tencent/bk-ci/src/booster/bk_dist/handler/common" "github.com/Tencent/bk-ci/src/booster/common/blog" ) const ( MaxWindowsCommandLength = 30000 + + appendEnvKey = "INCLUDE=" + osWindows = "windows" ) var ( @@ -56,6 +65,7 @@ type TaskCC struct { rewriteCrossArgs []string preProcessArgs []string serverSideArgs []string + pumpArgs []string // file names inputFile string @@ -64,9 +74,19 @@ type TaskCC struct { firstIncludeFile string pchFile string responseFile string + sourcedependfile string + pumpHeadFile string + + // forcedepend 是我们主动导出依赖文件，showinclude 是编译命令已经指定了导出依赖文件 + forcedepend bool + pumpremote bool + needcopypumpheadfile bool pchFileDesc *dcSDK.FileDesc + // for /showIncludes + showinclude bool + ForceLocalResponseFileKeys []string ForceLocalCppFileKeys []string } @@ -164,12 +184,409 @@ func (cc *TaskCC) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { }, nil } +func (cc *TaskCC) getIncludeExe() (string, error) { + blog.Debugf("cc: ready get include exe") + + target := "bk-includes" + if runtime.GOOS == osWindows { + target = "bk-includes.exe" + } + + includePath, err := dcUtil.CheckExecutable(target) + if err != nil { + // blog.Infof("cc: not found exe file with default path, info: %v", err) + + includePath, err = dcUtil.CheckFileWithCallerPath(target) + if err != nil { + blog.Errorf("cc: not found exe file with error: %v", err) + return includePath, err + } + } + absPath, err := filepath.Abs(includePath) + if err == nil { + includePath = absPath + } + includePath = dcUtil.QuoteSpacePath(includePath) + // blog.Infof("cc: got include exe file full path: %s", includePath) + + return includePath, nil +} + +func uniqArr(arr []string) []string { + newarr := make([]string, 0) + tempMap := make(map[string]bool, len(newarr)) + for _, v := range arr { + if tempMap[v] == false { + tempMap[v] = true + newarr = append(newarr, v) + } + } + + return newarr +} + +func (cc *TaskCC) analyzeIncludes(f string, workdir string) ([]*dcFile.Info, error) { + data, err := ioutil.ReadFile(f) + if err != nil { + return nil, err + } + + sep := "\n" + if runtime.GOOS == osWindows { + sep = "\r\n" + } + lines := strings.Split(string(data), sep) + includes := []*dcFile.Info{} + uniqlines := uniqArr(lines) + blog.Infof("cc: got %d uniq include file from file: %s", len(uniqlines), f) + + for _, l := range uniqlines { + if !filepath.IsAbs(l) { + l, _ = filepath.Abs(filepath.Join(workdir, l)) + } + fstat := dcFile.Stat(l) + if fstat.Exist() && !fstat.Basic().IsDir() { + includes = append(includes, fstat) + } else { + blog.Infof("cc: do not deal include file: %s in file:%s for not existed or is dir", l, f) + } + } + + return includes, nil +} + +func (cc *TaskCC) checkFstat(f string, workdir string) (*dcFile.Info, error) { + if !filepath.IsAbs(f) { + f, _ = filepath.Abs(filepath.Join(workdir, f)) + } + fstat := dcFile.Stat(f) + if fstat.Exist() && !fstat.Basic().IsDir() { + return fstat, nil + } + + return nil, nil +} + +func (cc *TaskCC) copyPumpHeadFile(workdir string) error { + blog.Infof("cc: copy pump head file: %s to: %s", cc.sourcedependfile, cc.pumpHeadFile) + data, err := ioutil.ReadFile(cc.sourcedependfile) + if err != nil { + blog.Warnf("cc: copy pump head failed to read depned file: %s with err:%v", cc.sourcedependfile, err) + return err + } + + sep := "\n" + if runtime.GOOS == osWindows { + sep = "\r\n" + } + lines := strings.Split(string(data), sep) + includes := []string{} + for _, l := range lines { + l = strings.Trim(l, " \r\n\\") + // TODO : the file path maybe contains space, should support this condition + fields := strings.Split(l, " ") + if len(fields) >= 1 { + for i, f := range fields { + if strings.HasSuffix(f, ".o:") { + continue + } + if !filepath.IsAbs(f) { + fields[i], _ = filepath.Abs(filepath.Join(workdir, f)) + } + includes = append(includes, fields[i]) + } + } + } + + blog.Infof("cc: copy pump head got %d uniq include file from file: %s", len(includes), cc.sourcedependfile) + + // TODO : save to cc.pumpHeadFile + newdata := strings.Join(includes, sep) + err = ioutil.WriteFile(cc.pumpHeadFile, []byte(newdata), os.ModePerm) + if err != nil { + blog.Warnf("cc: copy pump head failed to write file: %s with err:%v", cc.pumpHeadFile, err) + return err + } else { + blog.Infof("cc: copy pump head succeed to write file: %s", cc.pumpHeadFile) + } + + return nil +} + +// search all include files for this compile command +func (cc *TaskCC) Includes(responseFile string, args []string, workdir string, forcefresh bool) ([]*dcFile.Info, error) { + pumpdir := dcPump.PumpCacheDir(cc.sandbox.Env) + if pumpdir == "" { + pumpdir = dcUtil.GetPumpCacheDir() + } + + if !dcFile.Stat(pumpdir).Exist() { + if err := os.MkdirAll(pumpdir, os.ModePerm); err != nil { + return nil, err + } + } + + // TOOD : maybe we should pass responseFile to calc md5, to ensure unique + var err error + cc.pumpHeadFile, err = getPumpIncludeFile(pumpdir, "pump_heads", ".txt", args) + if err != nil { + blog.Errorf("cc: do includes get output file failed: %v", err) + return nil, err + } + + existed, fileSize, _, _ := dcFile.Stat(cc.pumpHeadFile).Batch() + if dcPump.IsPumpCache(cc.sandbox.Env) && !forcefresh && existed && fileSize > 0 { + return cc.analyzeIncludes(cc.pumpHeadFile, workdir) + } + + return nil, ErrorNoPumpHeadFile +} + +func (cc *TaskCC) forceDepend() error { + cc.sourcedependfile = makeTmpFileName(commonUtil.GetHandlerTmpDir(cc.sandbox), "cc_depend", ".d") + cc.sourcedependfile = strings.Replace(cc.sourcedependfile, "\\", "/", -1) + cc.addTmpFile(cc.sourcedependfile) + + cc.forcedepend = true + + return nil +} + +func (cc *TaskCC) inPumpBlack(responseFile string, args []string) (bool, error) { + // obtain black key set by booster + blackkeystr := cc.sandbox.Env.GetEnv(dcEnv.KeyExecutorPumpBlackKeys) + if blackkeystr != "" { + // blog.Infof("cc: got pump black key string: %s", blackkeystr) + blacklist := strings.Split(blackkeystr, dcEnv.CommonBKEnvSepKey) + if len(blacklist) > 0 { + for _, v := range blacklist { + if v != "" && strings.Contains(responseFile, v) { + blog.Infof("cc: found response %s is in pump blacklist", responseFile) + return true, nil + } + + for _, v1 := range args { + if strings.HasSuffix(v1, ".cpp") && strings.Contains(v1, v) { + blog.Infof("cc: found arg %s is in pump blacklist", v1) + return true, nil + } + } + } + } + } + + return false, nil +} + +// first error means real error when try pump, second is notify error +func (cc *TaskCC) trypump(command []string) (*dcSDK.BKDistCommand, error, error) { + blog.Infof("cc: trypump: %v", command) + + // TODO : !! ensureCompilerRaw changed the command slice, it maybe not we need !! + tstart := time.Now().Local() + responseFile, args, showinclude, sourcedependfile, objectfile, pchfile, err := ensureCompilerRaw(command, cc.sandbox.Dir) + if err != nil { + blog.Debugf("cc: pre execute ensure compiler failed %v: %v", args, err) + return nil, err, nil + } else { + blog.Infof("cc: after parse command, got responseFile:%s,sourcedepent:%s,objectfile:%s,pchfile:%s", + responseFile, sourcedependfile, objectfile, pchfile) + } + tend := time.Now().Local() + blog.Debugf("cc: trypump time record: %s for ensureCompilerRaw for rsp file:%s", tend.Sub(tstart), responseFile) + + // if sourcedependfile == "" { + // blog.Infof("cc: trypump not found depend file, do nothing") + // return nil, ErrorNoDependFile + // } + + tstart = tend + + _, err = scanArgs(args) + if err != nil { + blog.Debugf("cc: try pump not support, scan args %v: %v", args, err) + return nil, err, ErrorNotSupportRemote + } + + inblack, _ := cc.inPumpBlack(responseFile, args) + if inblack { + return nil, ErrorInPumpBlack, nil + } + + tend = time.Now().Local() + blog.Debugf("cc: trypump time record: %s for scanArgs for rsp file:%s", tend.Sub(tstart), responseFile) + tstart = tend + + if cc.sourcedependfile == "" { + if sourcedependfile != "" { + cc.sourcedependfile = sourcedependfile + } else { + // TODO : 我们可以主动加上 /showIncludes 参数得到依赖列表，生成一个临时的 cl.sourcedependfile 文件 + blog.Infof("cl: trypump not found depend file, try append it") + if cc.forceDepend() != nil { + return nil, ErrorNoDependFile, nil + } + } + } + cc.showinclude = showinclude + cc.needcopypumpheadfile = true + + cc.responseFile = responseFile + cc.pumpArgs = args + + includes, err := cc.Includes(responseFile, args, cc.sandbox.Dir, false) + + tend = time.Now().Local() + blog.Debugf("cc: trypump time record: %s for Includes for rsp file:%s", tend.Sub(tstart), responseFile) + tstart = tend + + // if sourcedependfile != "" { + // cc.needcopypumpheadfile = true + // } + + if err == nil { + blog.Infof("cc: parse command,got total %d includes files", len(includes)) + + // add pch file as input + if pchfile != "" { + // includes = append(includes, pchfile) + finfo, _ := cc.checkFstat(pchfile, cc.sandbox.Dir) + if finfo != nil { + includes = append(includes, finfo) + } + } + + // add response file as input + if responseFile != "" { + // includes = append(includes, responseFile) + finfo, _ := cc.checkFstat(responseFile, cc.sandbox.Dir) + if finfo != nil { + includes = append(includes, finfo) + } + } + + inputFiles := []dcSDK.FileDesc{} + // priority := dcSDK.MaxFileDescPriority + for _, f := range includes { + // existed, fileSize, modifyTime, fileMode := dcFile.Stat(f).Batch() + existed, fileSize, modifyTime, fileMode := f.Batch() + fpath := f.Path() + if !existed { + err := fmt.Errorf("input file %s not existed", fpath) + blog.Errorf("cc: %v", err) + return nil, err, nil + } + inputFiles = append(inputFiles, dcSDK.FileDesc{ + FilePath: fpath, + Compresstype: protocol.CompressLZ4, + FileSize: fileSize, + Lastmodifytime: modifyTime, + Md5: "", + Filemode: fileMode, + Targetrelativepath: filepath.Dir(fpath), + NoDuplicated: true, + // Priority: priority, + }) + // priority++ + // blog.Infof("cc: added include file:%s with modify time %d", fpath, modifyTime) + + blog.Debugf("cc: added include file:%s for object:%s", fpath, objectfile) + } + + results := []string{objectfile} + // add source depend file as result + if sourcedependfile != "" { + results = append(results, sourcedependfile) + } + + // set env which need append to remote + envs := []string{} + for _, v := range cc.sandbox.Env.Source() { + if strings.HasPrefix(v, appendEnvKey) { + envs = append(envs, v) + // set flag we hope append env, not overwrite + flag := fmt.Sprintf("%s=true", dcEnv.GetEnvKey(env.KeyRemoteEnvAppend)) + envs = append(envs, flag) + break + } + } + blog.Infof("cc: env which ready sent to remote:[%v]", envs) + + exeName := command[0] + params := command[1:] + blog.Infof("cc: parse command,server command:[%s %s],dir[%s]", + exeName, strings.Join(params, " "), cc.sandbox.Dir) + return &dcSDK.BKDistCommand{ + Commands: []dcSDK.BKCommand{ + { + WorkDir: cc.sandbox.Dir, + ExePath: "", + ExeName: exeName, + ExeToolChainKey: dcSDK.GetJsonToolChainKey(command[0]), + Params: params, + Inputfiles: inputFiles, + ResultFiles: results, + Env: envs, + }, + }, + CustomSave: true, + }, nil, nil + } + + tend = time.Now().Local() + blog.Debugf("cc: trypump time record: %s for return dcSDK.BKCommand for rsp file:%s", tend.Sub(tstart), responseFile) + + return nil, err, nil +} + +func (cc *TaskCC) isPumpActionNumSatisfied() (bool, error) { + minnum := dcPump.PumpMinActionNum(cc.sandbox.Env) + if minnum <= 0 { + return true, nil + } + + curbatchsize := 0 + strsize := cc.sandbox.Env.GetEnv(dcEnv.KeyExecutorTotalActionNum) + if strsize != "" { + size, err := strconv.Atoi(strsize) + if err != nil { + return true, err + } else { + curbatchsize = size + } + } + + blog.Infof("cc: check pump action num with min:%d: current batch num:%d", minnum, curbatchsize) + + return int32(curbatchsize) > minnum, nil +} + func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { blog.Infof("cc: start pre execute for: %v", command) // debugRecordFileName(fmt.Sprintf("cc: start pre execute for: %v", command)) cc.originArgs = command + + // ++ try with pump,only support windows now + if dcPump.SupportPump(cc.sandbox.Env) { + if satisfied, _ := cc.isPumpActionNumSatisfied(); satisfied { + req, err, notifyerr := cc.trypump(command) + if err != nil { + if notifyerr == ErrorNotSupportRemote { + blog.Warnf("cc: pre execute failed to try pump %v: %v", command, err) + return nil, err + } + } else { + // for debug + blog.Debugf("cc: after try pump, req: %+v", *req) + cc.pumpremote = true + return req, err + } + } + } + // -- + responseFile, args, err := ensureCompiler(command, cc.sandbox.Dir) if err != nil { blog.Warnf("cc: pre execute ensure compiler %v: %v", args, err) @@ -216,6 +633,12 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { // debugRecordFileName("preBuild begin") + if cc.forcedepend { + args = append(args, "-MD") + args = append(args, "-MF") + args = append(args, cc.sourcedependfile) + } + if err = cc.preBuild(args); err != nil { blog.Warnf("cc: pre execute pre-build %v: %v", args, err) return nil, err @@ -309,6 +732,10 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error { blog.Infof("cc: success done post execute for: %v", cc.originArgs) // set output to inputFile r.Results[0].OutputMessage = []byte(filepath.Base(cc.inputFile)) + // if remote succeed with pump,do not need copy head file + if cc.pumpremote { + cc.needcopypumpheadfile = false + } return nil } @@ -328,6 +755,11 @@ ERROREND: } } + if cc.pumpremote { + blog.Infof("cc: ready remove pump head file: %s after failed pump remote, generate it next time", cc.pumpHeadFile) + os.Remove(cc.pumpHeadFile) + } + return fmt.Errorf("cc: failed to remote execute, retcode %d, error message:%s, output message:%s", r.Results[0].RetCode, r.Results[0].ErrorMessage, @@ -339,6 +771,10 @@ func (cc *TaskCC) finalExecute([]string) { return } + if cc.needcopypumpheadfile { + cc.copyPumpHeadFile(cc.sandbox.Dir) + } + cc.cleanTmpFile() } @@ -427,7 +863,7 @@ func (cc *TaskCC) preBuild(args []string) error { } // quota result file if it's path contains space - if runtime.GOOS == "windows" { + if runtime.GOOS == osWindows { if hasSpace(cc.outputFile) && !strings.HasPrefix(cc.outputFile, "\"") { for index := range serverSideArgs { if strings.HasPrefix(serverSideArgs[index], "-o") { @@ -460,7 +896,7 @@ func (cc *TaskCC) addTmpFile(filename string) { func (cc *TaskCC) cleanTmpFile() { for _, filename := range cc.tmpFileList { if err := os.Remove(filename); err != nil { - blog.Warnf("cc: clean tmp file %s failed: %v", filename, err) + blog.Infof("cc: clean tmp file %s failed: %v", filename, err) } } } diff --git a/src/backend/booster/bk_dist/handler/ue4/cc/utils.go b/src/backend/booster/bk_dist/handler/ue4/cc/utils.go index 3c3be5e98d2..861672ec21d 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cc/utils.go +++ b/src/backend/booster/bk_dist/handler/ue4/cc/utils.go @@ -11,11 +11,13 @@ package cc import ( "bufio" + "crypto/md5" "fmt" "io/ioutil" "os" "path/filepath" "strings" + "sync" "time" dcFile "github.com/Tencent/bk-ci/src/booster/bk_dist/common/file" @@ -225,6 +227,126 @@ func replaceWithNextExclude(s string, old byte, new string, nextExcludes []byte) return string(targetslice) } +// ensure compiler exist in args. +func ensureCompilerRaw(args []string, workdir string) (string, []string, bool, string, string, string, error) { + responseFile := "" + sourcedependfile := "" + objectfile := "" + pchfile := "" + showinclude := false + if len(args) == 0 { + blog.Warnf("cc: ensure compiler got empty arg") + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, ErrorMissingOption + } + + if args[0] == "/" || args[0] == "@" || isSourceFile(args[0]) || isObjectFile(args[0]) { + return responseFile, append([]string{defaultCompiler}, args...), showinclude, sourcedependfile, objectfile, pchfile, nil + } + + for _, v := range args { + if strings.HasPrefix(v, "@") { + responseFile = strings.Trim(v[1:], "\"") + + data := "" + if responseFile != "" { + var err error + data, err = readResponse(responseFile, workdir) + if err != nil { + blog.Infof("cc: failed to read response file:%s,err:%v", responseFile, err) + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, err + } + } + // options, sources, err := parseArgument(data) + options, err := shlex.Split(replaceWithNextExclude(string(data), '\\', "\\\\", []byte{'"'})) + if err != nil { + blog.Infof("cc: failed to parse response file:%s,err:%v", responseFile, err) + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, err + } + + args = []string{args[0]} + args = append(args, options...) + + } else if v == "/showIncludes" { + showinclude = true + } + } + + firstinclude := true + for i := range args { + if strings.HasPrefix(args[i], "-MF") { + if len(args[i]) > 3 { + sourcedependfile = args[i][3:] + continue + } + + i++ + if i >= len(args) { + blog.Warnf("cc: scan args: no output file found after -MF") + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, ErrorMissingOption + } + sourcedependfile = args[i] + } else if strings.HasPrefix(args[i], "-o") { + // if -o just a prefix, the output file is also in this index, then skip the -o. + if len(args[i]) > 2 { + objectfile = args[i][2:] + blog.Infof("cc: got objectfile file:%s", objectfile) + continue + } + + i++ + if i >= len(args) { + blog.Warnf("cc: scan args: no output file found after -o") + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, ErrorMissingOption + } + objectfile = args[i] + blog.Infof("cc: got objectfile file:%s", objectfile) + } else if strings.HasPrefix(args[i], "-include-pch") { + firstinclude = false + if len(args[i]) > 12 { + pchfile = args[i][12:] + continue + } + + i++ + if i >= len(args) { + blog.Warnf("cc: scan args: no output file found after -include-pch") + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, ErrorMissingOption + } + pchfile = args[i] + } else if firstinclude && strings.HasPrefix(args[i], "-include") { + firstinclude = false + i++ + if i >= len(args) { + blog.Warnf("cc: scan args: no output file found after -include") + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, ErrorMissingOption + } + pchfile = args[i] + ".gch" + blog.Infof("cc: ready check gch file of %s", pchfile) + } + } + + if responseFile != "" && !filepath.IsAbs(responseFile) { + responseFile, _ = filepath.Abs(filepath.Join(workdir, responseFile)) + } + + if sourcedependfile != "" && !filepath.IsAbs(sourcedependfile) { + sourcedependfile, _ = filepath.Abs(filepath.Join(workdir, sourcedependfile)) + } + + if objectfile != "" && !filepath.IsAbs(objectfile) { + objectfile, _ = filepath.Abs(filepath.Join(workdir, objectfile)) + } + + if pchfile != "" && !filepath.IsAbs(pchfile) { + pchfile, _ = filepath.Abs(filepath.Join(workdir, pchfile)) + if !dcFile.Stat(pchfile).Exist() { + pchfile = "" + } + } + + return responseFile, args, showinclude, sourcedependfile, objectfile, pchfile, nil +} + // ensure compiler exist in args. // change "executor -c foo.c" -> "cc -c foo.c" func ensureCompiler(args []string, workdir string) (string, []string, error) { @@ -715,7 +837,7 @@ func scanArgs(args []string) (*ccArgs, error) { r.inputFile = arg continue } else { - blog.Infof("cc: arg[%s] is not source file", arg) + blog.Debugf("cc: arg[%s] is not source file", arg) } // if this file is end with .o, it must be the output file. @@ -951,6 +1073,43 @@ func makeTmpFile(tmpDir, prefix, ext string) (string, error) { return "", fmt.Errorf("cc: create tmp file failed: %s", target) } +func getPumpIncludeFile(tmpDir, prefix, ext string, args []string) (string, error) { + fullarg := strings.Join(args, " ") + md5str := md5.Sum([]byte(fullarg)) + target := filepath.Join(tmpDir, fmt.Sprintf("%s_%x%s", prefix, md5str, ext)) + + return target, nil +} + +func createFile(target string) error { + for i := 0; i < 3; i++ { + f, err := os.Create(target) + if err != nil { + blog.Errorf("cl: failed to create tmp file \"%s\": %s", target, err) + continue + } + + if err = f.Close(); err != nil { + blog.Errorf("cl: failed to close tmp file \"%s\": %s", target, err) + return err + } + + blog.Infof("cl: success to make tmp file \"%s\"", target) + return nil + } + + return fmt.Errorf("cl: create tmp file failed: %s", target) +} + +// only genegerate file name, do not create really +func makeTmpFileName(tmpDir, prefix, ext string) string { + pid := os.Getpid() + + return filepath.Join(tmpDir, + fmt.Sprintf("%s_%d_%s_%d%s", + prefix, pid, commonUtil.RandomString(8), time.Now().UnixNano(), ext)) +} + // Remove "-o" options from argument list. // // This is used when running the preprocessor, when we just want it to write @@ -1214,3 +1373,114 @@ func MakeCmdLine(args []string) string { } return s } + +// 根据 clang 命令，获取相应的 resource-dir +type clangResourceDirInfo struct { + clangcommandfullpath string + clangResourceDirpath string +} + +var ( + clangResourceDirlock sync.RWMutex + clangResourceDirs []clangResourceDirInfo +) + +func getResourceDir(cmd string) (string, error) { + var err error + exepfullath := cmd + if !filepath.IsAbs(cmd) { + exepfullath, err = dcUtil.CheckExecutable(cmd) + if err != nil { + return "", err + } + } + + // search from cache + clangResourceDirlock.RLock() + resourcedir := "" + for _, v := range clangResourceDirs { + if exepfullath == v.clangcommandfullpath { + resourcedir = v.clangResourceDirpath + clangResourceDirlock.RUnlock() + return resourcedir, nil + } + } + clangResourceDirlock.RUnlock() + + // try get resource-dir with clang exe path + clangResourceDirlock.Lock() + maxversion := "" + appended := false + defer func() { + // append to cache if not + if !appended { + clangResourceDirs = append(clangResourceDirs, clangResourceDirInfo{ + clangcommandfullpath: exepfullath, + clangResourceDirpath: maxversion, + }) + } + + clangResourceDirlock.Unlock() + }() + + // search from cache again, maybe append by others + for _, v := range clangResourceDirs { + if exepfullath == v.clangcommandfullpath { + resourcedir = v.clangResourceDirpath + appended = true + return resourcedir, nil + } + } + + // real compute resource-dir now + exedir := filepath.Dir(exepfullath) + exeparentdir := filepath.Dir(exedir) + foundclangdir := false + target := filepath.Join(exeparentdir, "lib", "clang") + if dcFile.Stat(target).Exist() { + blog.Infof("cc: found clang dir:%s by exe dir:%s", target, exepfullath) + foundclangdir = true + } else { + target = filepath.Join(exeparentdir, "lib64", "clang") + if dcFile.Stat(target).Exist() { + blog.Infof("cc: found clang dir:%s by exe dir:%s", target, exepfullath) + foundclangdir = true + } + } + + if !foundclangdir { + return resourcedir, fmt.Errorf("not found clang dir") + } + + // get all version dirs, and select the max + files, err := ioutil.ReadDir(target) + if err != nil { + blog.Warnf("failed to get version dirs from dir:%s", target) + return resourcedir, err + } + + versiondirs := []string{} + for _, file := range files { + if file.IsDir() { + nums := strings.Split(file.Name(), ".") + if len(nums) > 1 { + versiondirs = append(versiondirs, filepath.Join(target, file.Name())) + } + } + } + blog.Infof("cc: found all clang version dir:%v", versiondirs) + + if len(versiondirs) == 0 { + return resourcedir, fmt.Errorf("not found any clang's version dir") + } + + maxversion = versiondirs[0] + for _, v := range versiondirs { + if v > maxversion { + maxversion = v + } + } + + blog.Infof("cc: found final resource dir:%s by exe dir:%s", maxversion, exepfullath) + return maxversion, nil +} diff --git a/src/backend/booster/bk_dist/handler/ue4/cl/error.go b/src/backend/booster/bk_dist/handler/ue4/cl/error.go index 81d01e54ba0..5095fc6dc04 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cl/error.go +++ b/src/backend/booster/bk_dist/handler/ue4/cl/error.go @@ -27,4 +27,10 @@ var ( ErrorNotSupportYc = fmt.Errorf("/Yc must be local") ErrorNotSupportYcStart = fmt.Errorf("option start with /Yc must be local") ErrorNotSupportOutputStdout = fmt.Errorf("output to stdout, must be local") + ErrorNoPumpHeadFile = fmt.Errorf("pump head file not exist") + ErrorNoDependFile = fmt.Errorf("depend file not exist") + ErrorInvalidDependFile = fmt.Errorf("depend file invalid") + ErrorNotRemoteTask = fmt.Errorf("not remote task") + ErrorNotSupportRemote = fmt.Errorf("not support to remote execute") + ErrorInPumpBlack = fmt.Errorf("in pump black list") ) diff --git a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go index 01af2b321be..f2369456e00 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go @@ -10,12 +10,16 @@ package cl import ( + "bufio" "bytes" + "encoding/json" "fmt" + "io" "io/ioutil" "os" "path/filepath" "runtime" + "strconv" "strings" "time" @@ -41,6 +45,7 @@ const ( MaxWindowsCommandLength = 30000 appendEnvKey = "INCLUDE=" + osWindows = "windows" ) var ( @@ -99,6 +104,16 @@ type TaskCL struct { firstIncludeFile string pchFile string responseFile string + sourcedependfile string + pumpHeadFile string + + // forcedepend 是我们主动导出依赖文件，showinclude 是编译命令已经指定了导出依赖文件 + forcedepend bool + pumpremote bool + needcopypumpheadfile bool + + // how to save result file + customSave bool // to save preprocessed file content preprocessedBuffer []byte @@ -145,6 +160,10 @@ func NewCL() *TaskCL { } } +func (cl *TaskCL) SetDepend(f string) { + cl.sourcedependfile = f +} + // GetPreprocessedBuf return preprocessedErrorBuf func (cl *TaskCL) GetPreprocessedBuf() string { return cl.preprocessedErrorBuf @@ -289,7 +308,7 @@ func (cl *TaskCL) getIncludeExe() (string, error) { blog.Debugf("cl: ready get include exe") target := "bk-includes" - if runtime.GOOS == "windows" { + if runtime.GOOS == osWindows { target = "bk-includes.exe" } @@ -326,14 +345,14 @@ func uniqArr(arr []string) []string { return newarr } -func (cl *TaskCL) analyzeIncludes(f string, workdir string) ([]string, error) { +func (cl *TaskCL) analyzeIncludes(f string, workdir string) ([]*dcFile.Info, error) { data, err := ioutil.ReadFile(f) if err != nil { return nil, err } lines := strings.Split(string(data), "\r\n") - includes := []string{} + includes := []*dcFile.Info{} uniqlines := uniqArr(lines) blog.Infof("cl: got %d uniq include file from file: %s", len(uniqlines), f) @@ -343,7 +362,7 @@ func (cl *TaskCL) analyzeIncludes(f string, workdir string) ([]string, error) { } fstat := dcFile.Stat(l) if fstat.Exist() && !fstat.Basic().IsDir() { - includes = append(includes, l) + includes = append(includes, fstat) } else { blog.Infof("cl: do not deal include file: %s in file:%s for not existed or is dir", l, f) } @@ -352,8 +371,125 @@ func (cl *TaskCL) analyzeIncludes(f string, workdir string) ([]string, error) { return includes, nil } +func (cl *TaskCL) checkFstat(f string, workdir string) (*dcFile.Info, error) { + if !filepath.IsAbs(f) { + f, _ = filepath.Abs(filepath.Join(workdir, f)) + } + fstat := dcFile.Stat(f) + if fstat.Exist() && !fstat.Basic().IsDir() { + return fstat, nil + } + + return nil, nil +} + +type sourceDependenciesData struct { + Source string `json:"Source"` + ProvidedModule string `json:"ProvidedModule"` + PCH string `json:"PCH"` + Includes []string `json:"Includes"` +} + +type sourceDependencies struct { + Version string `json:"Version"` + Data sourceDependenciesData `json:"Data"` +} + +func (cl *TaskCL) copyPumpHeadFile(workdir string) error { + blog.Infof("cl: copy pump head file: %s to: %s", cl.sourcedependfile, cl.pumpHeadFile) + + // 只拷贝由加速编译预处理生成的依赖文件；非加速模式下生成的依赖文件不完整，去掉了系统文件 + if cl.inputFile == "" { + blog.Infof("cl: not found input file,so do not copy depend file: %s with err:%v", cl.sourcedependfile, ErrorNotRemoteTask) + return ErrorNotRemoteTask + } + + data, err := ioutil.ReadFile(cl.sourcedependfile) + if err != nil { + blog.Warnf("cl: copy pump head failed to read depend file: %s with err:%v", cl.sourcedependfile, err) + return err + } + + sep := "\n" + if runtime.GOOS == osWindows { + sep = "\r\n" + } + + includes := []string{} + if strings.HasSuffix(cl.sourcedependfile, ".json") { + var depend sourceDependencies + if err := json.Unmarshal(data, &depend); err == nil { + l := depend.Data.Source + if !filepath.IsAbs(l) { + l, _ = filepath.Abs(filepath.Join(workdir, l)) + } + includes = append(includes, l) + + for _, l := range depend.Data.Includes { + if !filepath.IsAbs(l) { + l, _ = filepath.Abs(filepath.Join(workdir, l)) + } + includes = append(includes, l) + } + } else { + blog.Warnf("cl: failed to resolve depend file: %s with err:%s", cl.sourcedependfile, err) + return err + } + } else { + lines := strings.Split(string(data), sep) + for _, l := range lines { + l = strings.Trim(l, " \r\n\\") + // // TODO : the file path maybe contains space, should support this condition + // fields := strings.Split(l, " ") + // if len(fields) >= 1 { + // for i, f := range fields { + // if strings.HasSuffix(f, ".o:") { + // continue + // } + // if !filepath.IsAbs(f) { + // fields[i], _ = filepath.Abs(filepath.Join(workdir, f)) + // } + // includes = append(includes, fields[i]) + // } + // } + if !filepath.IsAbs(l) { + l, _ = filepath.Abs(filepath.Join(workdir, l)) + } + includes = append(includes, l) + } + } + + // copy input file + if cl.inputFile != "" { + l := cl.inputFile + if !filepath.IsAbs(l) { + l, _ = filepath.Abs(filepath.Join(workdir, l)) + } + includes = append(includes, l) + } + + blog.Infof("cl: copy pump head got %d uniq include file from file: %s", len(includes), cl.sourcedependfile) + + if len(includes) == 0 { + blog.Warnf("cl: depend file: %s data:[%s] is invalid", cl.sourcedependfile, string(data)) + return ErrorInvalidDependFile + } + + // TODO : save to cc.pumpHeadFile + newdata := strings.Join(includes, sep) + err = ioutil.WriteFile(cl.pumpHeadFile, []byte(newdata), os.ModePerm) + if err != nil { + blog.Warnf("cl: copy pump head failed to write file: %s with err:%v", cl.pumpHeadFile, err) + return err + } else { + blog.Infof("cl: copy pump head succeed to write file: %s", cl.pumpHeadFile) + } + + return nil +} + // search all include files for this compile command -func (cl *TaskCL) Includes(responseFile string, args []string, workdir string, forcefresh bool) ([]string, error) { +func (cl *TaskCL) Includes(responseFile string, args []string, workdir string, forcefresh bool) ([]*dcFile.Info, error) { pumpdir := dcPump.PumpCacheDir(cl.sandbox.Env) if pumpdir == "" { pumpdir = dcUtil.GetPumpCacheDir() @@ -366,65 +502,59 @@ func (cl *TaskCL) Includes(responseFile string, args []string, workdir string, f } // TOOD : maybe we should pass responseFile to calc md5, to ensure unique - outputFile, err := getPumpIncludeFile(pumpdir, "pump_heads", ".txt", args) + var err error + cl.pumpHeadFile, err = getPumpIncludeFile(pumpdir, "pump_heads", ".txt", args) if err != nil { blog.Errorf("cl: do includes get output file failed: %v", err) return nil, err } - existed, fileSize, _, _ := dcFile.Stat(outputFile).Batch() + existed, fileSize, _, _ := dcFile.Stat(cl.pumpHeadFile).Batch() if dcPump.IsPumpCache(cl.sandbox.Env) && !forcefresh && existed && fileSize > 0 { - return cl.analyzeIncludes(outputFile, workdir) + return cl.analyzeIncludes(cl.pumpHeadFile, workdir) } - err = createFile(outputFile) - if err != nil { - return nil, err - } - - execName, err := cl.getIncludeExe() - if err != nil { - return nil, err - } - - // do not delete to use when cache mode - // cl.addTmpFile(outputFile) + return nil, ErrorNoPumpHeadFile +} - execArgs := []string{"-Xtbs", "--verbose=0", "--driver-mode=cl"} - if responseFile != "" { - execArgs = append(execArgs, "-Xtbs") - farg := fmt.Sprintf("--cmd_file=%s", responseFile) - execArgs = append(execArgs, farg) - } else { - execArgs = append(execArgs, args[1:]...) - } +func (cl *TaskCL) forceDepend() error { + cl.sourcedependfile = makeTmpFileName(commonUtil.GetHandlerTmpDir(cl.sandbox), "cl_depend", ".txt") + cl.addTmpFile(cl.sourcedependfile) - // TODO : ensure all absolute file path - sandbox := cl.sandbox.Fork() + cl.forcedepend = true + // args = append(args, "/showIncludes") - output, err := os.OpenFile(outputFile, os.O_WRONLY, 0666) - if err != nil { - blog.Errorf("cc: failed to open output file \"%s\" when pre-processing: %v", outputFile, err) - return nil, err - } - defer func() { - _ = output.Close() - }() + return nil +} - sandbox.Stdout = output - var errBuf bytes.Buffer - sandbox.Stderr = &errBuf +func (cl *TaskCL) inPumpBlack(responseFile string, args []string) (bool, error) { + // obtain black key set by booster + blackkeystr := cl.sandbox.Env.GetEnv(dcEnv.KeyExecutorPumpBlackKeys) + if blackkeystr != "" { + // blog.Infof("cl: got pump black key string: %s", blackkeystr) + blacklist := strings.Split(blackkeystr, dcEnv.CommonBKEnvSepKey) + if len(blacklist) > 0 { + for _, v := range blacklist { + if v != "" && strings.Contains(responseFile, v) { + blog.Infof("cl: found response %s is in pump blacklist", responseFile) + return true, nil + } - blog.Infof("cl: ready to do Includes %s %s", execName, strings.Join(execArgs, " ")) - if _, err = sandbox.ExecCommand(execName, execArgs...); err != nil { - blog.Warnf("cl: failed to do Includes %s %s with error:%v", execName, strings.Join(execArgs, " "), err) - return nil, err + for _, v1 := range args { + if strings.HasSuffix(v1, ".cpp") && strings.Contains(v1, v) { + blog.Infof("cl: found arg %s is in pump blacklist", v1) + return true, nil + } + } + } + } } - return cl.analyzeIncludes(outputFile, workdir) + return false, nil } -func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error) { +// first error means real error when try pump, second is notify error +func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error, error) { blog.Infof("cl: trypump: %v", command) // TODO : !! ensureCompilerRaw changed the command slice, it maybe not we need !! @@ -432,29 +562,53 @@ func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error) { responseFile, args, showinclude, sourcedependfile, objectfile, pchfile, err := ensureCompilerRaw(command, cl.sandbox.Dir) if err != nil { blog.Debugf("cl: pre execute ensure compiler failed %v: %v", args, err) - return nil, err + return nil, err, nil } else { blog.Infof("cl: after parse command, got responseFile:%s,sourcedepent:%s,objectfile:%s,pchfile:%s", responseFile, sourcedependfile, objectfile, pchfile) } tend := time.Now().Local() blog.Debugf("cl: trypump time record: %s for ensureCompilerRaw for rsp file:%s", tend.Sub(tstart), responseFile) + tstart = tend + // check whether support remote execute _, err = scanArgs(args) if err != nil { blog.Debugf("cl: try pump not support, scan args %v: %v", args, err) - return nil, err + return nil, err, ErrorNotSupportRemote + } + + inblack, _ := cl.inPumpBlack(responseFile, args) + if inblack { + return nil, ErrorInPumpBlack, nil } + if cl.sourcedependfile == "" { + if sourcedependfile != "" { + cl.sourcedependfile = sourcedependfile + } else { + // TODO : 我们可以主动加上 /showIncludes 参数得到依赖列表，生成一个临时的 cl.sourcedependfile 文件 + blog.Infof("cl: trypump not found depend file, try append it") + if cl.forceDepend() != nil { + return nil, ErrorNoDependFile, nil + } + } + } + cl.showinclude = showinclude + cl.needcopypumpheadfile = true + tend = time.Now().Local() blog.Debugf("cl: trypump time record: %s for scanArgs for rsp file:%s", tend.Sub(tstart), responseFile) tstart = tend cl.responseFile = responseFile - cl.showinclude = showinclude cl.pumpArgs = args + // if cl.sourcedependfile != "" { + // cl.needcopypumpheadfile = true + // } + includes, err := cl.Includes(responseFile, args, cl.sandbox.Dir, false) tend = time.Now().Local() @@ -466,37 +620,48 @@ func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error) { // add pch file as input if pchfile != "" { - includes = append(includes, pchfile) + // includes = append(includes, pchfile) + finfo, _ := cl.checkFstat(pchfile, cl.sandbox.Dir) + if finfo != nil { + includes = append(includes, finfo) + } } // add response file as input if responseFile != "" { - includes = append(includes, responseFile) + // includes = append(includes, responseFile) + finfo, _ := cl.checkFstat(responseFile, cl.sandbox.Dir) + if finfo != nil { + includes = append(includes, finfo) + } } inputFiles := []dcSDK.FileDesc{} // priority := dcSDK.MaxFileDescPriority for _, f := range includes { - existed, fileSize, modifyTime, fileMode := dcFile.Stat(f).Batch() + // TODO : 前面的 cl.Includes 已经调用过 dcFile.Stat 了，考虑将结果传过来，避免再次调用 + // existed, fileSize, modifyTime, fileMode := dcFile.Stat(f).Batch() + existed, fileSize, modifyTime, fileMode := f.Batch() + fpath := f.Path() if !existed { - err := fmt.Errorf("input response file %s not existed", f) + err := fmt.Errorf("input response file %s not existed", fpath) blog.Errorf("%v", err) - return nil, err + return nil, err, nil } inputFiles = append(inputFiles, dcSDK.FileDesc{ - FilePath: f, + FilePath: fpath, Compresstype: protocol.CompressLZ4, FileSize: fileSize, Lastmodifytime: modifyTime, Md5: "", Filemode: fileMode, - Targetrelativepath: filepath.Dir(f), + Targetrelativepath: filepath.Dir(fpath), NoDuplicated: true, // Priority: priority, }) // priority++ - blog.Debugf("cl: added include file:%s for object:%s", f, objectfile) + blog.Debugf("cl: added include file:%s for object:%s", fpath, objectfile) } results := []string{objectfile} @@ -522,6 +687,8 @@ func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error) { params := command[1:] blog.Infof("cl: parse command,server command:[%s %s],dir[%s]", exeName, strings.Join(params, " "), cl.sandbox.Dir) + + cl.customSave = true return &dcSDK.BKDistCommand{ Commands: []dcSDK.BKCommand{ { @@ -536,13 +703,35 @@ func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error) { }, }, CustomSave: true, - }, nil + }, nil, nil } tend = time.Now().Local() blog.Debugf("cl: trypump time record: %s for return dcSDK.BKCommand for rsp file:%s", tend.Sub(tstart), responseFile) - return nil, err + return nil, err, nil +} + +func (cl *TaskCL) isPumpActionNumSatisfied() (bool, error) { + minnum := dcPump.PumpMinActionNum(cl.sandbox.Env) + if minnum <= 0 { + return true, nil + } + + curbatchsize := 0 + strsize := cl.sandbox.Env.GetEnv(dcEnv.KeyExecutorTotalActionNum) + if strsize != "" { + size, err := strconv.Atoi(strsize) + if err != nil { + return true, err + } else { + curbatchsize = size + } + } + + blog.Infof("cl: check pump action num with min:%d: current batch num:%d", minnum, curbatchsize) + + return int32(curbatchsize) > minnum, nil } func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { @@ -554,11 +743,17 @@ func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { // ++ try with pump,only support windows now if dcPump.SupportPump(cl.sandbox.Env) { - req, err := cl.trypump(command) - if err != nil { - blog.Warnf("cl: pre execute failed to try pump %v: %v", command, err) - } else { - return req, err + if satisfied, _ := cl.isPumpActionNumSatisfied(); satisfied { + req, err, notifyerr := cl.trypump(command) + if err != nil { + if notifyerr == ErrorNotSupportRemote { + blog.Warnf("cl: pre execute failed to try pump %v: %v", command, err) + return nil, err + } + } else { + cl.pumpremote = true + return req, err + } } } // -- @@ -616,6 +811,10 @@ func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { tstart := time.Now().Local() + if cl.forcedepend { + args = append(args, "/showIncludes") + } + if err = cl.preBuild(args); err != nil { blog.Debugf("cl: pre execute pre-build %v: %v", args, err) return nil, err @@ -687,6 +886,7 @@ func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { }) } + cl.customSave = true return &dcSDK.BKDistCommand{ Commands: []dcSDK.BKCommand{ { @@ -733,7 +933,7 @@ func (cl *TaskCL) postExecute(r *dcSDK.BKDistResult) error { } // by tomtian 20201224,to ensure existed result file - if resultfilenum == 0 { + if resultfilenum == 0 && cl.customSave { blog.Warnf("cl: not found result file for: %v", cl.originArgs) goto ERROREND } @@ -743,14 +943,25 @@ func (cl *TaskCL) postExecute(r *dcSDK.BKDistResult) error { if r.Results[0].RetCode == 0 { blog.Infof("cl: success done post execute for: %v", cl.originArgs) if cl.showinclude { - if !dcPump.SupportPump(cl.sandbox.Env) { + // if !dcPump.SupportPump(cl.sandbox.Env) { + if cl.preprocessedErrorBuf != "" { // simulate output with preprocessed error output r.Results[0].OutputMessage = []byte(cl.preprocessedErrorBuf) } + } else if cl.forcedepend { + if cl.preprocessedErrorBuf != "" { + cl.parseOutput(cl.preprocessedErrorBuf) + } } else { // simulate output with inputFile r.Results[0].OutputMessage = []byte(filepath.Base(cl.inputFile)) } + + // if remote succeed with pump,do not need copy head file + if cl.pumpremote { + cl.needcopypumpheadfile = false + } + return nil } @@ -770,10 +981,9 @@ ERROREND: } } - // if remote failed with pump mode, we need refresh the header list - if dcPump.SupportPump(cl.sandbox.Env) && dcPump.IsPumpCache(cl.sandbox.Env) { - blog.Infof("cl: ready to fresh pump header files for cmd: [%v]", cl.pumpArgs) - _, _ = cl.Includes(cl.responseFile, cl.pumpArgs, cl.sandbox.Dir, true) + if cl.pumpremote { + blog.Infof("cl: ready remove pump head file: %s after failed pump remote, generate it next time", cl.pumpHeadFile) + os.Remove(cl.pumpHeadFile) } return fmt.Errorf("cl: failed to remote execute, retcode %d, error message:%s, output message:%s", @@ -787,6 +997,10 @@ func (cl *TaskCL) finalExecute([]string) { return } + if cl.needcopypumpheadfile { + cl.copyPumpHeadFile(cl.sandbox.Dir) + } + cl.cleanTmpFile() } @@ -850,7 +1064,7 @@ func (cl *TaskCL) preBuild(args []string) error { } // quota result file if it's path contains space - if runtime.GOOS == "windows" { + if runtime.GOOS == osWindows { if hasSpace(cl.outputFile) && !strings.HasPrefix(cl.outputFile, "\"") { for index := range serverSideArgs { if strings.HasPrefix(serverSideArgs[index], "/Fo") { @@ -881,7 +1095,7 @@ func (cl *TaskCL) addTmpFile(filename string) { func (cl *TaskCL) cleanTmpFile() { for _, filename := range cl.tmpFileList { if err := os.Remove(filename); err != nil { - blog.Warnf("cl: clean tmp file %s failed: %v", filename, err) + blog.Debugf("cl: clean tmp file %s failed: %v", filename, err) } } } @@ -968,7 +1182,7 @@ func (cl *TaskCL) doPreProcess(args []string, inputFile string) (string, []byte, if !savetomemroy { output, err := os.OpenFile(outputFile, os.O_WRONLY, 0666) if err != nil { - blog.Errorf("cc: failed to open output file \"%s\" when pre-processing: %v", outputFile, err) + blog.Errorf("cl: failed to open output file \"%s\" when pre-processing: %v", outputFile, err) return "", nil, err } defer func() { @@ -989,7 +1203,7 @@ func (cl *TaskCL) doPreProcess(args []string, inputFile string) (string, []byte, return "", nil, err } blog.Infof("cl: success to execute pre-process and get %s: %s", outputFile, strings.Join(newArgs, " ")) - if cl.showinclude { + if cl.showinclude || cl.forcedepend { cl.preprocessedErrorBuf = errBuf.String() } @@ -1066,3 +1280,75 @@ func (cl *TaskCL) needSaveResponseFile(args []string) (bool, string, error) { return false, "", nil } + +// copied from clfilter handle +func (cl *TaskCL) parseOutput(s string) (string, error) { + blog.Debugf("cl: start parse output: %s", s) + + output := make([]string, 0, 0) + includes := make([]string, 0, 0) + + reader := bufio.NewReader(strings.NewReader(s)) + var line string + var err error + for { + line, err = reader.ReadString('\n') + if err != nil && err != io.EOF { + break + } + + // Process the line here. + // Note: including file: Runtime\Core\Public\HAL/ThreadHeartBeat.h + // Note: including file: C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.31.31103\INCLUDE\sal.h + columns := strings.Split(line, ":") + if len(columns) == 3 { + includefile := strings.Trim(columns[2], " \r\n") + if !filepath.IsAbs(includefile) { + includefile, _ = filepath.Abs(filepath.Join(cl.sandbox.Dir, includefile)) + } + existed, _, _, _ := dcFile.Stat(includefile).Batch() + if existed { + includes = append(includes, includefile) + } else { + blog.Infof("cl: includefile [%s] not existed", includefile) + output = append(output, line) + } + } else if len(columns) == 4 { + includefile := columns[2] + ":" + columns[3] + includefile = strings.Trim(includefile, " \r\n") + existed, _, _, _ := dcFile.Stat(includefile).Batch() + if existed { + includes = append(includes, includefile) + } else { + blog.Infof("cl: includefile [%s] not existed", includefile) + output = append(output, line) + } + } else { + output = append(output, line) + } + + if err != nil { + break + } + } + blog.Debugf("cl: got output: [%v], includes:[%v]", output, includes) + + // save includes to cl.sourcedependfile + if len(includes) > 0 { + f, err := os.Create(cl.sourcedependfile) + if err != nil { + blog.Errorf("cl: create file %s error: [%s]", cl.sourcedependfile, err.Error()) + } else { + defer func() { + _ = f.Close() + }() + _, err := f.Write([]byte(strings.Join(includes, "\r\n"))) + if err != nil { + blog.Errorf("cl: save depend file [%s] error: [%s]", cl.sourcedependfile, err.Error()) + return strings.Join(output, "\n"), err + } + } + } + + return strings.Join(output, ""), nil +} diff --git a/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go b/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go index aec1d35d928..0b7f8574132 100644 --- a/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go @@ -187,6 +187,8 @@ func (cf *TaskCLFilter) preExecute(command []string) (*dcSDK.BKDistCommand, erro blog.Infof("cf: after pre execute, got depend file: [%s], cl cmd:[%s]", cf.dependentFile, cf.cldArgs) cf.clhandle.InitSandbox(cf.sandbox) + cf.clhandle.SetDepend(cf.dependentFile) + return cf.clhandle.PreExecute(args) } diff --git a/src/backend/booster/bk_dist/monitor/pkg/monitor.go b/src/backend/booster/bk_dist/monitor/pkg/monitor.go index 24b13f187c1..77409de003c 100644 --- a/src/backend/booster/bk_dist/monitor/pkg/monitor.go +++ b/src/backend/booster/bk_dist/monitor/pkg/monitor.go @@ -23,6 +23,7 @@ import ( dcUtil "github.com/Tencent/bk-ci/src/booster/bk_dist/common/util" "github.com/Tencent/bk-ci/src/booster/bk_dist/monitor/types" "github.com/Tencent/bk-ci/src/booster/common/blog" + "github.com/shirou/gopsutil/process" ) const ( @@ -31,6 +32,8 @@ const ( DefautInterval = 20000 + BlockSecondsWhenRecover = 240 + MacroKillTree = "${BK_KILL_TREE}" ) @@ -117,36 +120,57 @@ func (m *Monitor) runRule(r types.Rule) { if len(r.RecoverCmds) > 0 { if v, ok := r.RecoverCmds[string(outmsg)]; ok { - for _, c := range v { - blog.Infof("monitor: ready run recover cmd:[%s]", c) - exitcode, outmsg, errmsg, err = m.runCommand(c) - if exitcode != 0 { - blog.Errorf("monitor: failed to execute cmd:[%s] with exit code:%d, err:%v", c, exitcode, err) - return - } - blog.Infof("monitor: succeed to execute cmd:[%s] with exit code:%d,output:%s,errmsg:%s, err:%v", - c, exitcode, outmsg, errmsg, err) - } + // for _, c := range v { + // blog.Infof("monitor: ready run recover cmd:[%s]", c) + // exitcode, outmsg, errmsg, err = m.runCommand(c) + // if exitcode != 0 { + // blog.Errorf("monitor: failed to execute cmd:[%s] with exit code:%d, err:%v", c, exitcode, err) + // return + // } + // blog.Infof("monitor: succeed to execute cmd:[%s] with exit code:%d,output:%s,errmsg:%s, err:%v", + // c, exitcode, outmsg, errmsg, err) + // } + go m.runRecoverCommands(v) + blog.Infof("monitor: ready sleep %d second to wait execute recover commands", BlockSecondsWhenRecover) + time.Sleep(BlockSecondsWhenRecover * time.Second) } else { trimkey := strings.Trim(string(outmsg), "\r\n \t") if trimkey != string(outmsg) { if v, ok := r.RecoverCmds[trimkey]; ok { - for _, c := range v { - blog.Infof("monitor: ready run recover cmd:[%s]", c) - exitcode, outmsg, errmsg, err = m.runCommand(c) - if exitcode != 0 { - blog.Errorf("monitor: failed to execute cmd:[%s] with exit code:%d, err:%v", c, exitcode, err) - return - } - blog.Infof("monitor: succeed to execute cmd:[%s] with exit code:%d,output:%s,errmsg:%s, err:%v", - c, exitcode, outmsg, errmsg, err) - } + // for _, c := range v { + // blog.Infof("monitor: ready run recover cmd:[%s]", c) + // exitcode, outmsg, errmsg, err = m.runCommand(c) + // if exitcode != 0 { + // blog.Errorf("monitor: failed to execute cmd:[%s] with exit code:%d, err:%v", c, exitcode, err) + // return + // } + // blog.Infof("monitor: succeed to execute cmd:[%s] with exit code:%d,output:%s,errmsg:%s, err:%v", + // c, exitcode, outmsg, errmsg, err) + // } + go m.runRecoverCommands(v) + blog.Infof("monitor: ready sleep %d second to wait execute recover commands", BlockSecondsWhenRecover) + time.Sleep(BlockSecondsWhenRecover * time.Second) } } } } } +func (m *Monitor) runRecoverCommands(cmds []string) { + blog.Infof("monitor: ready run recover cmds:%+v", cmds) + + for _, c := range cmds { + blog.Infof("monitor: ready run recover cmd:[%s]", c) + exitcode, outmsg, errmsg, err := m.runCommand(c) + if exitcode != 0 { + blog.Errorf("monitor: failed to execute cmd:[%s] with exit code:%d, err:%v", c, exitcode, err) + return + } + blog.Infof("monitor: succeed to execute cmd:[%s] with exit code:%d,output:%s,errmsg:%s, err:%v", + c, exitcode, outmsg, errmsg, err) + } +} + func (m *Monitor) runCommand(c string) (int, []byte, []byte, error) { if strings.Contains(c, MacroKillTree) { return m.killTree(c) @@ -169,18 +193,63 @@ func (m *Monitor) killTree(c string) (int, []byte, []byte, error) { if len(procs) == 0 { blog.Infof("monitor: not found any process with name:%s", args[1]) + return 0, []byte(""), []byte(""), nil } + // 如果是 bk-dist-monitor 自己拉起来的，还需要主动释放下，避免留下脏进程 + proc1 := []*process.Process{} for _, v := range procs { + newp, err := m.searchRootProc(v) + if err == nil { + proc1 = append(proc1, newp) + } else { + proc1 = append(proc1, v) + } + } + + for _, v := range proc1 { name, _ := v.Name() blog.Infof("monitor: ready kill process %s %d", name, int32(v.Pid)) KillChildren(v) - _ = v.Kill() + err := KillProcess(v) + if err != nil { + blog.Infof("monitor: kill process %s %d failed with err:%v", name, int32(v.Pid), err) + } } return 0, []byte(""), []byte(""), nil } +// 如果有祖先进程是 bk-dist-monitor 自己拉起来的，则返回该祖先，否则返回自己 +func (m *Monitor) searchRootProc(p *process.Process) (*process.Process, error) { + selfpid := os.Getpid() + newp := p + ppid, err := newp.Ppid() + if err != nil { + blog.Infof("monitor: get parent pid with error:%v", err) + return p, err + } + + for { + if ppid == int32(selfpid) { + return newp, nil + } + + newp, err = process.NewProcess(ppid) + if err != nil { + blog.Infof("monitor: get parent process with pid:%d with error:%v", ppid, err) + return p, err + } + + ppid, err = newp.Ppid() + if err != nil { + blog.Infof("monitor: get parent pid with error:%v", err) + return p, err + } + } + +} + func (m *Monitor) initRules() error { f, err := m.getRulesFile() if err != nil { diff --git a/src/backend/booster/bk_dist/monitor/pkg/util.go b/src/backend/booster/bk_dist/monitor/pkg/util.go index a5d8642270d..49251b11a09 100644 --- a/src/backend/booster/bk_dist/monitor/pkg/util.go +++ b/src/backend/booster/bk_dist/monitor/pkg/util.go @@ -13,6 +13,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "github.com/Tencent/bk-ci/src/booster/bk_dist/common/flock" "github.com/Tencent/bk-ci/src/booster/bk_dist/common/util" @@ -22,13 +23,32 @@ import ( "github.com/shirou/gopsutil/process" ) +func KillProcess(p *process.Process) error { + var err error + if runtime.GOOS != "windows" { + err = p.Kill() + } else { + targetp, err := os.FindProcess(int(p.Pid)) + if err == nil { + err = targetp.Kill() + return err + } + } + + return err +} + // KillChildren kill all process children of given process func KillChildren(p *process.Process) { children, err := p.Children() if err == nil && len(children) > 0 { for _, v := range children { KillChildren(v) - _ = v.Kill() + err = KillProcess(v) + if err != nil { + name, _ := v.Name() + blog.Infof("monitor: kill child process %s %d failed with err:%v", name, int32(v.Pid), err) + } } } } diff --git a/src/backend/booster/bk_dist/ubttool/common/types.go b/src/backend/booster/bk_dist/ubttool/common/types.go index db9fde670b8..2ff04d111ed 100644 --- a/src/backend/booster/bk_dist/ubttool/common/types.go +++ b/src/backend/booster/bk_dist/ubttool/common/types.go @@ -32,6 +32,10 @@ type Action struct { FollowIndex []string `json:"followindex"` // follower index which depend on this Running bool `json:"running"` Finished bool `json:"finished"` + //其它详细信息 + IsCompile bool + ModulePath string + Desc string } // UE4Action to desc ubt actions diff --git a/src/backend/booster/bk_dist/ubttool/pkg/executor.go b/src/backend/booster/bk_dist/ubttool/pkg/executor.go index 92fd0f98748..472e83a539c 100644 --- a/src/backend/booster/bk_dist/ubttool/pkg/executor.go +++ b/src/backend/booster/bk_dist/ubttool/pkg/executor.go @@ -65,7 +65,7 @@ func (d *Executor) Update() { func (d *Executor) Run(fullargs []string, workdir string) (int, string, error) { blog.Infof("ubtexecutor: command [%s] begins", strings.Join(fullargs, " ")) for i, v := range fullargs { - blog.Infof("ubtexecutor: arg[%d] : [%s]", i, v) + blog.Debugf("ubtexecutor: arg[%d] : [%s]", i, v) } defer blog.Infof("ubtexecutor: command [%s] finished", strings.Join(fullargs, " ")) diff --git a/src/backend/booster/bk_dist/ubttool/pkg/ubttool.go b/src/backend/booster/bk_dist/ubttool/pkg/ubttool.go index 5bc8b2d5be6..317ab137321 100644 --- a/src/backend/booster/bk_dist/ubttool/pkg/ubttool.go +++ b/src/backend/booster/bk_dist/ubttool/pkg/ubttool.go @@ -40,10 +40,11 @@ import ( ) const ( - OSWindows = "windows" - MaxWaitSecs = 10800 - TickSecs = 30 - DefaultJobs = 240 // ok for most machines + OSWindows = "windows" + MaxWaitSecs = 10800 + TickSecs = 30 + DefaultJobs = 240 // ok for most machines + ActionDescMaxSize = 50 DevOPSProcessTreeKillKey = "DEVOPS_DONT_KILL_PROCESS_TREE" ) @@ -63,6 +64,7 @@ func NewUBTTool(flagsparam *common.Flags, config dcSDK.ControllerConfig) *UBTToo finished: false, actionchan: nil, executor: NewExecutor(), + moduleselected: make(map[string]int, 0), } } @@ -86,6 +88,8 @@ type UBTTool struct { maxjobs int32 finished bool + moduleselected map[string]int + actionchan chan common.Actionresult booster *pkg.Booster @@ -148,6 +152,12 @@ func (h *UBTTool) runActions() error { // execute actions here h.allactions = all.Actions + + // parse actions firstly + h.analyzeActions(h.allactions) + // for debug + blog.Debugf("UBTTool: all actions:%+v", h.allactions) + // readyactions includes actions which no depend err = h.getReadyActions() if err != nil { @@ -229,38 +239,117 @@ func (h *UBTTool) executeActions() error { } } +// // to simply print log +// func getActionDesc(cmd, arg string) string { +// // _, _ = fmt.Fprintf(os.Stdout, "cmd %s arg %s\n", cmd, arg) + +// exe := filepath.Base(cmd) +// targetsuffix := []string{} +// switch exe { +// case "cl.exe", "cl-filter.exe", "clang.exe", "clang++.exe", "clang", "clang++": +// targetsuffix = []string{".cpp", ".c", ".response\"", ".response"} +// break +// case "lib.exe", "link.exe", "link-filter.exe": +// targetsuffix = []string{".dll", ".lib", ".response\"", ".response"} +// default: +// return exe +// } + +// args, _ := shlex.Split(replaceWithNextExclude(arg, '\\', "\\\\", []byte{'"'})) +// if len(args) == 1 { +// argbase := strings.TrimRight(filepath.Base(arg), "\"") +// return fmt.Sprintf("%s %s", exe, argbase) +// } else { +// for _, v := range args { +// for _, s := range targetsuffix { +// if strings.HasSuffix(v, s) { +// vtrime := strings.TrimRight(v, "\"") +// return fmt.Sprintf("%s %s", exe, vtrime) +// } +// } +// } +// } + +// return exe +// } + // to simply print log -func getActionDesc(cmd, arg string) string { - // _, _ = fmt.Fprintf(os.Stdout, "cmd %s arg %s\n", cmd, arg) - - exe := filepath.Base(cmd) - targetsuffix := []string{} - switch exe { - case "cl.exe", "cl-filter.exe", "clang.exe", "clang++.exe", "clang", "clang++": - targetsuffix = []string{".cpp", ".c", ".response\"", ".response"} - break - case "lib.exe", "link.exe", "link-filter.exe": - targetsuffix = []string{".dll", ".lib", ".response\"", ".response"} - default: - return exe +func (h *UBTTool) analyzeActions(actions []common.Action) error { + for i, v := range actions { + cmd := v.Cmd + arg := v.Arg + exe := filepath.Base(cmd) + targetsuffix := []string{} + needAnalyzeArg := true + switch exe { + case "cl.exe", "cl-filter.exe", "clang.exe", "clang++.exe", "clang", "clang++": + targetsuffix = []string{".cpp", ".c", ".response\"", ".response"} + actions[i].IsCompile = true + break + case "lib.exe", "link.exe", "link-filter.exe": + targetsuffix = []string{".dll", ".lib", ".response\"", ".response"} + default: + needAnalyzeArg = false + arglen := ActionDescMaxSize + if arglen > len(arg) { + arglen = len(arg) + } + actions[i].Desc = fmt.Sprintf("%s %s...", exe, arg[0:arglen]) + } + + if !needAnalyzeArg { + continue + } + + args, _ := shlex.Split(replaceWithNextExclude(arg, '\\', "\\\\", []byte{'"'})) + if len(args) == 1 { + argbase := strings.TrimRight(filepath.Base(arg), "\"") + actions[i].Desc = fmt.Sprintf("%s %s", exe, argbase) + actions[i].ModulePath = filepath.Dir(arg) + } else { + foundSuffix := false + for _, v := range args { + foundSuffix = false + for _, s := range targetsuffix { + if strings.HasSuffix(v, s) { + vtrime := strings.TrimRight(v, "\"") + actions[i].Desc = fmt.Sprintf("%s %s", exe, vtrime) + actions[i].ModulePath = filepath.Dir(v) + foundSuffix = true + break + } + } + if foundSuffix { + break + } + } + + if !foundSuffix { + arglen := ActionDescMaxSize + if arglen > len(arg) { + arglen = len(arg) + } + actions[i].Desc = fmt.Sprintf("%s %s...", exe, arg[0:arglen]) + } + } } - args, _ := shlex.Split(replaceWithNextExclude(arg, '\\', "\\\\", []byte{'"'})) - if len(args) == 1 { - argbase := strings.TrimRight(filepath.Base(arg), "\"") - return fmt.Sprintf("%s %s", exe, argbase) - } else { - for _, v := range args { - for _, s := range targetsuffix { - if strings.HasSuffix(v, s) { - vtrime := strings.TrimRight(v, "\"") - return fmt.Sprintf("%s %s", exe, vtrime) + totalcompilenum := 0 + for _, v := range actions { + if v.IsCompile { + totalcompilenum++ + if v.ModulePath != "" { + if _, ok := h.moduleselected[v.ModulePath]; !ok { + h.moduleselected[v.ModulePath] = 0 } } } } - return exe + env.SetEnv(env.KeyExecutorTotalActionNum, strconv.Itoa(totalcompilenum)) + blog.Infof("UBTTool: set total action num with: %s=%d", env.KeyExecutorTotalActionNum, totalcompilenum) + + return nil } func (h *UBTTool) selectActionsToExecute() error { @@ -276,8 +365,8 @@ func (h *UBTTool) selectActionsToExecute() error { h.readyactions[index].Running = true h.runningnumber++ _, _ = fmt.Fprintf(os.Stdout, "[bk_ubt_tool] [%d/%d] %s\n", - h.finishednumber+h.runningnumber, len(h.allactions), - getActionDesc(h.readyactions[index].Cmd, h.readyactions[index].Arg)) + h.finishednumber+h.runningnumber, len(h.allactions), h.readyactions[index].Desc) + // getActionDesc(h.readyactions[index].Cmd, h.readyactions[index].Arg)) go h.executeOneAction(h.readyactions[index], h.actionchan) } @@ -287,6 +376,7 @@ func (h *UBTTool) selectActionsToExecute() error { func (h *UBTTool) selectReadyAction() int { index := -1 followers := -1 + // select ready action which is not running and has most followers if h.flags.MostDepentFirst { for i := range h.readyactions { @@ -309,6 +399,9 @@ func (h *UBTTool) selectReadyAction() int { if index >= 0 { blog.Infof("UBTTool: selected global index %s with %d followers", h.readyactions[index].Index, followers) + if h.readyactions[index].IsCompile && h.readyactions[index].ModulePath != "" { + h.moduleselected[h.readyactions[index].ModulePath]++ + } } return index } @@ -394,7 +487,7 @@ func (h *UBTTool) onActionFinished(index string, exitcode int) error { blog.Infof("UBTTool: running : %d, finished : %d, total : %d", h.runningnumber, h.finishednumber, len(h.allactions)) if h.finishednumber >= int32(len(h.allactions)) { h.finishednumberlock.Unlock() - blog.Infof("UBTTool: finishend") + blog.Infof("UBTTool: finishend,module selected:%+v", h.moduleselected) h.finished = true return nil } diff --git a/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_handler.go b/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_handler.go index e8a786b77f6..28d23cabb40 100644 --- a/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_handler.go +++ b/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_handler.go @@ -367,7 +367,7 @@ func (r *CommonRemoteHandler) ExecuteSendFile( } d := time.Now().Sub(t) if d > 200*time.Millisecond { - blog.Infof("TCP Connect to long to server(%s): %s", server.Server, d.String()) + blog.Debugf("TCP Connect to long to server(%s): %s", server.Server, d.String()) } defer func() { _ = client.Close() @@ -438,7 +438,7 @@ func (r *CommonRemoteHandler) ExecuteCheckCache( } d := time.Now().Sub(t) if d > 200*time.Millisecond { - blog.Infof("TCP Connect to long to server(%s): %s", server.Server, d.String()) + blog.Debugf("TCP Connect to long to server(%s): %s", server.Server, d.String()) } defer func() { _ = client.Close() diff --git a/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_protocol.go b/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_protocol.go index a29aef706c9..c296f6b6a1e 100644 --- a/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_protocol.go +++ b/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_protocol.go @@ -319,6 +319,7 @@ func encodeCommonDispatchReq(req *dcSDK.BKDistCommand) ([]protocol.Message, erro targetrelativepath := f.Targetrelativepath filemode := f.Filemode linkTarget := f.LinkTarget + modifytime := f.Lastmodifytime if size < 0 { pbcommand.Inputfiles = append(pbcommand.Inputfiles, &protocol.PBFileDesc{ @@ -330,6 +331,7 @@ func encodeCommonDispatchReq(req *dcSDK.BKDistCommand) ([]protocol.Message, erro Targetrelativepath: &targetrelativepath, Filemode: &filemode, Linktarget: []byte(linkTarget), + Modifytime: &modifytime, }) continue } @@ -353,6 +355,7 @@ func encodeCommonDispatchReq(req *dcSDK.BKDistCommand) ([]protocol.Message, erro Targetrelativepath: &targetrelativepath, Filemode: &filemode, Linktarget: []byte(linkTarget), + Modifytime: &modifytime, }) filemessages = append(filemessages, m) @@ -822,6 +825,7 @@ func encodeSendFileReq(req *dcSDK.BKDistFileSender, sandbox *syscall.Sandbox) ([ targetrelativepath := f.Targetrelativepath filemode := f.Filemode linkTarget := f.LinkTarget + modifytime := f.Lastmodifytime if size <= 0 { pbbody.Inputfiles = append(pbbody.Inputfiles, &protocol.PBFileDesc{ @@ -833,6 +837,7 @@ func encodeSendFileReq(req *dcSDK.BKDistFileSender, sandbox *syscall.Sandbox) ([ Targetrelativepath: &targetrelativepath, Filemode: &filemode, Linktarget: []byte(linkTarget), + Modifytime: &modifytime, }) continue } @@ -856,8 +861,11 @@ func encodeSendFileReq(req *dcSDK.BKDistFileSender, sandbox *syscall.Sandbox) ([ Targetrelativepath: &targetrelativepath, Filemode: &filemode, Linktarget: []byte(linkTarget), + Modifytime: &modifytime, }) + // blog.Infof("encode send files add file(%s) modify time(%d)", fullpath, modifytime) + filemessages = append(filemessages, m) filebuflen += compressedsize } @@ -996,9 +1004,12 @@ func encodeCheckCacheReq(req *dcSDK.BKDistFileSender, sandbox *syscall.Sandbox) } pbbody.Params = append(pbbody.Params, &protocol.PBCacheParam{ - Name: []byte(filepath.Base(f.FilePath)), - Md5: []byte(md5), - Target: []byte(fullpath), + Name: []byte(filepath.Base(f.FilePath)), + Md5: []byte(md5), + Target: []byte(fullpath), + Filemode: &f.Filemode, + Linktarget: []byte(f.LinkTarget), + Modifytime: &f.Lastmodifytime, }) } diff --git a/src/backend/booster/bk_dist/worker/pkg/client/tcp_client.go b/src/backend/booster/bk_dist/worker/pkg/client/tcp_client.go index e2e199c0714..e61412b4228 100644 --- a/src/backend/booster/bk_dist/worker/pkg/client/tcp_client.go +++ b/src/backend/booster/bk_dist/worker/pkg/client/tcp_client.go @@ -59,10 +59,10 @@ func (c *TCPClient) Connect(server string) error { c.conn, err = net.DialTCP("tcp", nil, resolvedserver) d := time.Now().Sub(t) if d > 50*time.Millisecond { - blog.Infof("TCP Dail to long gt50 to server(%s): %s", resolvedserver, d.String()) + blog.Debugf("TCP Dail to long gt50 to server(%s): %s", resolvedserver, d.String()) } if d > 200*time.Millisecond { - blog.Infof("TCP Dail to long gt200 to server(%s): %s", resolvedserver, d.String()) + blog.Debugf("TCP Dail to long gt200 to server(%s): %s", resolvedserver, d.String()) } if err != nil { @@ -71,7 +71,7 @@ func (c *TCPClient) Connect(server string) error { } blog.Debugf("succeed to connect to server [%s] ", server) - blog.Infof("succeed to establish connection [%s] ", c.ConnDesc()) + // blog.Infof("succeed to establish connection [%s] ", c.ConnDesc()) // not sure whether it can imporve performance err = c.conn.SetNoDelay(false) @@ -302,7 +302,7 @@ func sendMessages(client *TCPClient, messages []protocol.Message) error { // Close close conn func (c *TCPClient) Close() error { - blog.Infof("ready close connection [%v] ", c.ConnDesc()) + blog.Debugf("ready close connection [%v] ", c.ConnDesc()) return c.conn.Close() } diff --git a/src/backend/booster/bk_dist/worker/pkg/cmd_handler/handler4filecache.go b/src/backend/booster/bk_dist/worker/pkg/cmd_handler/handler4filecache.go index 7c452ad388b..78c0d7b5961 100644 --- a/src/backend/booster/bk_dist/worker/pkg/cmd_handler/handler4filecache.go +++ b/src/backend/booster/bk_dist/worker/pkg/cmd_handler/handler4filecache.go @@ -11,6 +11,7 @@ package pbcmd import ( "fmt" + "os" "strconv" "sync" "time" @@ -99,6 +100,8 @@ func (h *Handle4FileCache) Handle(client *protocol.TCPClient, name := string(param.GetName()) md5 := string(param.GetMd5()) target := string(param.GetTarget()) + filemode := param.GetFilemode() + modifytime := param.GetModifytime() if name == "" || md5 == "" || target == "" { blog.Warnf("file cache: try check file cache with name(%s) md5(%s) target(%s) failed: "+ @@ -108,10 +111,10 @@ func (h *Handle4FileCache) Handle(client *protocol.TCPClient, } wg.Add(1) - go func(i int, n, m, t string) { + go func(i int, n, m, t string, fm uint32, mt int64) { defer wg.Done() - result[i] = h.searchCacheAndGetFileSaved(n, m, t) - }(index, name, md5, target) + result[i] = h.searchCacheAndGetFileSaved(n, m, t, fm, mt) + }(index, name, md5, target, filemode, modifytime) } wg.Wait() @@ -133,14 +136,35 @@ func (h *Handle4FileCache) Handle(client *protocol.TCPClient, return nil } -func (h *Handle4FileCache) searchCacheAndGetFileSaved(name, md5, target string) *dcProtocol.PBCacheResult { +func (h *Handle4FileCache) searchCacheAndGetFileSaved(name, md5, target string, + filemode uint32, modifytime int64) *dcProtocol.PBCacheResult { // TODO (tomtian) : if target existed and md5 same, do nothing // maybe should check whether file mode is changed + existed := false + defer func() { + if existed { + if filemode > 0 { + blog.Infof("ready set cached file[%s] with filemode[%d]", target, filemode) + if err := os.Chmod(target, os.FileMode(filemode)); err != nil { + blog.Warnf("chmod file %s to file-mode %s failed: %v", target, os.FileMode(filemode), err) + } + } + + if modifytime > 0 { + blog.Infof("ready set cached file[%s] modify time [%d]", target, modifytime) + if err := os.Chtimes(target, time.Now(), time.Unix(0, modifytime)); err != nil { + blog.Warnf("Chtimes file %s to time %s failed: %v", target, time.Unix(0, modifytime), err) + } + } + } + }() + finfo := dcFile.Stat(target) if finfo.Exist() { oldmd5, _ := finfo.Md5() if oldmd5 == md5 { - blog.Infof("file cache: target file[%s] with md5(%s) existed, do nothing", target, md5) + blog.Infof("file cache: target file[%s] with md5(%s) existed, set filemode and modify time now", target, md5) + existed = true return encodeRsp(dcProtocol.PBCacheStatus_SUCCESS, "") } } @@ -166,6 +190,7 @@ func (h *Handle4FileCache) searchCacheAndGetFileSaved(name, md5, target string) return encodeRsp(dcProtocol.PBCacheStatus_ERRORWHILESAVING, err.Error()) } + existed = true blog.Infof("file cache: success to hit file cache with name(%s) md5(%s) and save to %s", name, md5, target) return encodeRsp(dcProtocol.PBCacheStatus_SUCCESS, "") } diff --git a/src/backend/booster/bk_dist/worker/pkg/protocol/pb_protocol.go b/src/backend/booster/bk_dist/worker/pkg/protocol/pb_protocol.go index d93af750bbf..59975e53c07 100644 --- a/src/backend/booster/bk_dist/worker/pkg/protocol/pb_protocol.go +++ b/src/backend/booster/bk_dist/worker/pkg/protocol/pb_protocol.go @@ -484,6 +484,26 @@ func saveFile( inputfile = targetname } + + // !! set file attribute after rename + // TODO : change filemode and modifytime if new file created here + if newfilesaved { + filemode := rf.GetFilemode() + if filemode > 0 { + blog.Infof("get file[%s] filemode[%d]", inputfile, filemode) + if err = os.Chmod(inputfile, os.FileMode(filemode)); err != nil { + blog.Warnf("chmod file %s to file-mode %s failed: %v", inputfile, os.FileMode(filemode), err) + } + } + + modifytime := rf.GetModifytime() + if modifytime > 0 { + blog.Infof("get file[%s] modify time [%d]", inputfile, modifytime) + if err = os.Chtimes(inputfile, time.Now(), time.Unix(0, modifytime)); err != nil { + blog.Warnf("Chtimes file %s to time %s failed: %v", inputfile, time.Unix(0, modifytime), err) + } + } + } }() if rf.GetCompressedsize() > 0 { @@ -495,11 +515,11 @@ func saveFile( return "", err } - filemode := rf.GetFilemode() - if filemode > 0 { - blog.Infof("get file[%s] filemode[%d]", inputfile, filemode) - _ = os.Chmod(inputfile, os.FileMode(filemode)) - } + // filemode := rf.GetFilemode() + // if filemode > 0 { + // blog.Infof("get file[%s] filemode[%d]", inputfile, filemode) + // _ = os.Chmod(inputfile, os.FileMode(filemode)) + // } break // case protocol.PBCompressType_LZO: // // decompress with lzox1 firstly @@ -559,13 +579,13 @@ func saveFile( return "", err } - filemode := rf.GetFilemode() - if filemode > 0 { - blog.Infof("get file[%s] filemode[%d]", inputfile, filemode) - if err = os.Chmod(inputfile, os.FileMode(filemode)); err != nil { - blog.Warnf("chmod file %s to file-mode %s failed: %v", inputfile, os.FileMode(filemode), err) - } - } + // filemode := rf.GetFilemode() + // if filemode > 0 { + // blog.Infof("get file[%s] filemode[%d]", inputfile, filemode) + // if err = os.Chmod(inputfile, os.FileMode(filemode)); err != nil { + // blog.Warnf("chmod file %s to file-mode %s failed: %v", inputfile, os.FileMode(filemode), err) + // } + // } break default: return "", fmt.Errorf("unknown compress type [%s]", rf.GetCompresstype()) diff --git a/src/backend/booster/common/types/bcs_types.go b/src/backend/booster/common/types/bcs_types.go index b512a4c6750..e12254ee42f 100644 --- a/src/backend/booster/common/types/bcs_types.go +++ b/src/backend/booster/common/types/bcs_types.go @@ -149,3 +149,16 @@ type ResourceItem struct { Mem string `json:"memory,omitempty"` Storage string `json:"storage,omitempty"` } + +// Image define a worker iamge on devops +type Image struct { + Name string `json:"param_name"` + Value string `json:"param_value"` + ProjectWhitelist []string `json:"visual_range"` +} + +// WorkerImage define a worker iamge on devops +type WorkerImage struct { + Mesos []Image `json:"mesos"` + K8s []Image `json:"k8s"` +} diff --git a/src/backend/booster/common/version/version.go b/src/backend/booster/common/version/version.go index 7aa5586b4de..b601bf4fde8 100644 --- a/src/backend/booster/common/version/version.go +++ b/src/backend/booster/common/version/version.go @@ -10,12 +10,8 @@ package version import ( - "bytes" "fmt" "runtime" - - "github.com/opesun/goquery" - "github.com/opesun/goquery/exp/html" ) var ( @@ -51,15 +47,3 @@ func GetVersion() string { runtime.Version(), Version, Tag, BuildTime, GitHash) return version } - -func Search(buf *bytes.Buffer, n *goquery.Node) { - if n == nil { - return - } - if n.Type == html.TextNode { - fmt.Fprintf(buf, "%v", n.Data) - } - for _, v := range n.Child { - Search(buf, &goquery.Node{v}) - } -} diff --git a/src/backend/booster/gateway/pkg/api/v1/api.go b/src/backend/booster/gateway/pkg/api/v1/api.go index 0b2cf769f5c..8b2998782fc 100644 --- a/src/backend/booster/gateway/pkg/api/v1/api.go +++ b/src/backend/booster/gateway/pkg/api/v1/api.go @@ -59,7 +59,13 @@ func initDCCActions() { Params: nil, Handler: api.NoLimit(distcc.ListTask), }) - + // distcc worker images + api.RegisterV1Action(api.Action{ + Verb: "GET", + Path: "/distcc/resource/images", + Params: nil, + Handler: api.NoLimit(distcc.ListWorkerImages), + }) // distcc project api.RegisterV1Action(api.Action{ Verb: "GET", @@ -247,6 +253,14 @@ func initDistTaskActions() { Handler: api.NoLimit(disttask.ListClientVersion), }) + // disttask worker images + api.RegisterV1Action(api.Action{ + Verb: "GET", + Path: "/disttask/resource/images", + Params: nil, + Handler: api.NoLimit(disttask.ListWorkerImages), + }) + // disttask task api.RegisterV1Action(api.Action{ Verb: "GET", diff --git a/src/backend/booster/gateway/pkg/api/v1/distcc/handler.go b/src/backend/booster/gateway/pkg/api/v1/distcc/handler.go index a64707d2b54..677f7f49963 100644 --- a/src/backend/booster/gateway/pkg/api/v1/distcc/handler.go +++ b/src/backend/booster/gateway/pkg/api/v1/distcc/handler.go @@ -10,6 +10,7 @@ package distcc import ( + "encoding/json" "fmt" "io/ioutil" "strconv" @@ -47,6 +48,34 @@ func ListTask(req *restful.Request, resp *restful.Response) { api.ReturnRest(&api.RestResponse{Resp: resp, Data: taskList, Extra: map[string]interface{}{"length": length}}) } +// ListWorkerImages handle the http request for listing worker images +func ListWorkerImages(req *restful.Request, resp *restful.Response) { + args := req.Request.URL.Query() + queueName := args.Get("queue_name") + + filePath := "./data/DistccWorkerList.json" + result := commonTypes.WorkerImage{ + Mesos: make([]commonTypes.Image, 0, 100), + K8s: make([]commonTypes.Image, 0, 100), + } + + data, _ := ioutil.ReadFile(filePath) + err := json.Unmarshal([]byte(data), &result) + if err != nil { + api.ReturnRest(&api.RestResponse{Resp: resp, Message: err.Error()}) + } + + switch queueName { + case "shenzhen", "shanghai", "chengdu", "tianjin": + api.ReturnRest(&api.RestResponse{Resp: resp, Data: result.Mesos}) + case "K8S://gd": + api.ReturnRest(&api.RestResponse{Resp: resp, Data: result.K8s}) + default: + message := fmt.Sprintf("unknown queue name : %s", queueName) + api.ReturnRest(&api.RestResponse{Resp: resp, Message: message}) + } +} + // ListProject handle the http request for listing project with conditions. func ListProject(req *restful.Request, resp *restful.Response) { opts, err := getListOptions(req, "PROJECT") diff --git a/src/backend/booster/gateway/pkg/api/v1/disttask/handler.go b/src/backend/booster/gateway/pkg/api/v1/disttask/handler.go index f417eef974b..1ac463d7ba4 100644 --- a/src/backend/booster/gateway/pkg/api/v1/disttask/handler.go +++ b/src/backend/booster/gateway/pkg/api/v1/disttask/handler.go @@ -10,7 +10,7 @@ package disttask import ( - "bytes" + "encoding/json" "fmt" "io/ioutil" "sort" @@ -18,13 +18,11 @@ import ( "strings" "github.com/Tencent/bk-ci/src/booster/bk_dist/common/types" - "github.com/opesun/goquery" "github.com/Tencent/bk-ci/src/booster/common/blog" "github.com/Tencent/bk-ci/src/booster/common/codec" commonMySQL "github.com/Tencent/bk-ci/src/booster/common/mysql" commonTypes "github.com/Tencent/bk-ci/src/booster/common/types" - "github.com/Tencent/bk-ci/src/booster/common/version" "github.com/Tencent/bk-ci/src/booster/gateway/pkg/api" "github.com/Tencent/bk-ci/src/booster/server/pkg/engine" "github.com/Tencent/bk-ci/src/booster/server/pkg/engine/disttask" @@ -34,31 +32,50 @@ import ( // ListClientVersion handle the http request for listing client version func ListClientVersion(req *restful.Request, resp *restful.Response) { - url := version.DisttaskRepo - res, err := goquery.ParseUrl(url) + filePath := "./data/VersionList.txt" + f, err := ioutil.ReadFile(filePath) if err != nil { - blog.Errorf("list client version failed, err: %v", err) - api.ReturnRest(&api.RestResponse{Resp: resp, ErrCode: commonTypes.ServerErrListVersionFailed, Message: err.Error()}) - } - result := make([]string, 0, 100) - nodes := res.Find("a") - for _, v := range nodes { - buf := &bytes.Buffer{} - version.Search(buf, v) - s := buf.String() - if strings.HasPrefix(s, "install_v") { - i1 := strings.Index(s, "_v") - i2 := strings.Index(s, ".sh") - if i2 < i1 { - continue - } - result = append(result, s[i1+1:i2]) - } + blog.Error("List Version error:(%v)", err) + api.ReturnRest(&api.RestResponse{Resp: resp, Message: err.Error()}) + } + s := string(f) + if strings.HasSuffix(s, "\n") { + s = s[:len(s)-1] } + result := strings.Split(s, "\n") + sort.Sort(sort.Reverse(sort.StringSlice(result))) api.ReturnRest(&api.RestResponse{Resp: resp, Data: result}) } +// ListWorkerImages handle the http request for listing worker images +func ListWorkerImages(req *restful.Request, resp *restful.Response) { + args := req.Request.URL.Query() + queueName := args.Get("queue_name") + + filePath := "./data/DisttaskWorkerList.json" + + result := commonTypes.WorkerImage{ + Mesos: make([]commonTypes.Image, 0, 100), + K8s: make([]commonTypes.Image, 0, 100), + } + data, _ := ioutil.ReadFile(filePath) + err := json.Unmarshal([]byte(data), &result) + if err != nil { + api.ReturnRest(&api.RestResponse{Resp: resp, Message: err.Error()}) + } + + switch queueName { + case "shenzhen", "shanghai", "chengdu", "tianjin": + api.ReturnRest(&api.RestResponse{Resp: resp, Data: result.Mesos}) + case "K8S://gd": + api.ReturnRest(&api.RestResponse{Resp: resp, Data: result.K8s}) + default: + message := fmt.Sprintf("unknown queue name : %s", queueName) + api.ReturnRest(&api.RestResponse{Resp: resp, Message: message}) + } +} + // ListTask handle the http request for listing task with conditions. func ListTask(req *restful.Request, resp *restful.Response) { opts, err := getListOptions(req, "TASK") diff --git a/src/backend/booster/server/pkg/api/v2/handler.go b/src/backend/booster/server/pkg/api/v2/handler.go index 3d4c65f2c8b..9fa88ac2120 100644 --- a/src/backend/booster/server/pkg/api/v2/handler.go +++ b/src/backend/booster/server/pkg/api/v2/handler.go @@ -244,7 +244,7 @@ func getTaskInfo(taskID string) (*RespTaskInfo, error) { if tb.Status.Status == engine.TaskStatusStaging { rank, err = defaultManager.GetTaskRank(taskID) if err != nil { - blog.Errorf("get apply param: get task(q%s) rank from engine(%s) queue(%s) failed: %v", + blog.Warnf("get apply param: get task(%s) rank from engine(%s) queue(%s) failed: %v", taskID, tb.Client.EngineName.String(), tb.Client.QueueName, err) rank = 0 } diff --git a/src/backend/booster/server/pkg/manager/normal/manager.go b/src/backend/booster/server/pkg/manager/normal/manager.go index 74af834070b..34da279321a 100644 --- a/src/backend/booster/server/pkg/manager/normal/manager.go +++ b/src/backend/booster/server/pkg/manager/normal/manager.go @@ -158,7 +158,7 @@ func (m *manager) GetTaskRank(taskID string) (int, error) { rank, err := qg.GetQueue(tb.Client.QueueName).Rank(taskID) if err != nil { - blog.Errorf("manager: try getting task rank, get task(%s) rank from engine(%s) queue(%s) failed: %v", + blog.Warnf("manager: try getting task rank, get task(%s) rank from engine(%s) queue(%s) failed: %v", taskID, tb.Client.EngineName, tb.Client.QueueName, err) return -1, err } @@ -632,6 +632,15 @@ func generateTaskID(egnName string, projectID string) string { taskIDFormat, egnName, projectID, time.Now().Unix(), strings.ToLower(util.RandomString(taskIDRandomLength))) } +//IsOldTaskType check if the task id type is old +func IsOldTaskType(id string) bool { + idx := strings.LastIndex(id, "-") + if idx == len(id)-taskIDRandomLength-1 { //old task Id + return true + } + return false +} + func ip2long(ipStr string) (uint32, error) { ip := net.ParseIP(ipStr) if ip == nil { diff --git a/src/backend/booster/server/pkg/resource/crm/manager.go b/src/backend/booster/server/pkg/resource/crm/manager.go index 654b2ae5cef..118f36c80e8 100644 --- a/src/backend/booster/server/pkg/resource/crm/manager.go +++ b/src/backend/booster/server/pkg/resource/crm/manager.go @@ -24,6 +24,7 @@ import ( commonMySQL "github.com/Tencent/bk-ci/src/booster/common/mysql" "github.com/Tencent/bk-ci/src/booster/server/config" "github.com/Tencent/bk-ci/src/booster/server/pkg/engine" + "github.com/Tencent/bk-ci/src/booster/server/pkg/manager/normal" rsc "github.com/Tencent/bk-ci/src/booster/server/pkg/resource" op "github.com/Tencent/bk-ci/src/booster/server/pkg/resource/crm/operator" "github.com/Tencent/bk-ci/src/booster/server/pkg/resource/crm/operator/k8s" @@ -32,7 +33,7 @@ import ( ) //DefaultNamespace define default namespace for resourcemanager -const DefaultNamespace = "disttask" +//const DefaultNamespace = "disttask" // NewResourceManager get a new container resource manager. func NewResourceManager( @@ -403,12 +404,22 @@ func (rm *resourceManager) recover() error { rm.registeredResourceMap = make(map[string]*resource, 1000) for _, r := range rl { + if rm.conf.Operator == config.CRMOperatorK8S && rm.conf.InstanceType != nil { + isBelongToRm := false + for _, ist := range rm.conf.InstanceType { + if ist.Group == r.param.City && ist.Platform == r.param.Platform { + isBelongToRm = true + } + } + if !isBelongToRm { + continue + } + } rm.registeredResourceMap[r.resourceID] = r if r.noReadyInstance <= 0 { continue } - // recover the no-ready records rm.nodeInfoPool.RecoverNoReadyBlock(r.resourceBlockKey, r.noReadyInstance) blog.Infof("crm: recover no-ready-instance(%d) from resource(%s)", r.noReadyInstance, r.resourceID) @@ -418,13 +429,10 @@ func (rm *resourceManager) recover() error { } func (rm *resourceManager) sync() { - if rm.conf.BcsNamespace == "" { - rm.conf.BcsNamespace = DefaultNamespace - } nodeInfoList, err := rm.operator.GetResource(rm.conf.BcsClusterID) if err != nil { - blog.Errorf("crm: sync resource failed: %v", err) + blog.Errorf("crm: sync resource failed, clusterId(%s): %v", rm.conf.BcsClusterID, err) return } @@ -670,8 +678,8 @@ func (rm *resourceManager) getServiceInfo(resourceID, user string) (*op.ServiceI } info, err := rm.operator.GetServerStatus(rm.conf.BcsClusterID, rm.handlerMap[user].GetNamespace(), targetID) if err != nil { - blog.Errorf("crm: get service info for resource(%s) target(%s) user(%s) failed: %v", - resourceID, targetID, user, err) + blog.Errorf("crm: get service info for resource(%s) target(%s) user(%s) namespace(%s) failed: %v", + resourceID, targetID, user, rm.handlerMap[user].GetNamespace(), err) return nil, err } @@ -949,7 +957,7 @@ func (rm *resourceManager) release(resourceID, user string) error { r, err := rm.getResources(resourceID) if err != nil { - blog.Errorf("crm: try releasing service, get resource(%s) for user(%s) failed: %v", + blog.Warnf("crm: try releasing service, get resource(%s) for user(%s) failed: %v", resourceID, user, err) return err } @@ -1151,10 +1159,13 @@ func (hwu *handlerWithUser) GetNamespace() string { if hwu.mgr.conf.BcsNamespace != "" { return hwu.mgr.conf.BcsNamespace } - return DefaultNamespace + return hwu.user } func (hwu *handlerWithUser) resourceID(id string) string { + if normal.IsOldTaskType(id) { //old task Id + return strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s-%s", hwu.user, id)), "_", "-") + } return strings.ReplaceAll(strings.ToLower(id), "_", "-") } diff --git a/src/backend/booster/server/pkg/resource/crm/operator/k8s/operator.go b/src/backend/booster/server/pkg/resource/crm/operator/k8s/operator.go index e2ccd9ac30e..5c835950ef9 100644 --- a/src/backend/booster/server/pkg/resource/crm/operator/k8s/operator.go +++ b/src/backend/booster/server/pkg/resource/crm/operator/k8s/operator.go @@ -363,8 +363,10 @@ func (o *operator) getFederationTotalNum(url string, ist config.InstanceType) (F func (o *operator) getFederationResource(clusterID string) ([]*op.NodeInfo, error) { nodeInfoList := make([]*op.NodeInfo, 0, 1000) - ns := o.conf.BcsNamespace - url := fmt.Sprintf(bcsAPIFederatedURI, o.conf.BcsAPIPool.GetAddress(), clusterID, ns) + if o.conf.BcsNamespace == "" { + return nil, fmt.Errorf("crm: get federation resource request failed clusterID(%s): namespace is nil", clusterID) + } + url := fmt.Sprintf(bcsAPIFederatedURI, o.conf.BcsAPIPool.GetAddress(), clusterID, o.conf.BcsNamespace) for _, ist := range o.conf.InstanceType { result, err := o.getFederationTotalNum(url, ist) if err != nil { @@ -379,8 +381,8 @@ func (o *operator) getFederationResource(clusterID string) ([]*op.NodeInfo, erro } totalIst := float64(result.Data.Total) nodeInfoList = append(nodeInfoList, &op.NodeInfo{ - IP: clusterID + "-" + ns + "-" + ist.Platform + "-" + ist.Group, - Hostname: clusterID + "-" + ns + "-" + ist.Platform + "-" + ist.Group, + IP: clusterID + "-" + o.conf.BcsNamespace + "-" + ist.Platform + "-" + ist.Group, + Hostname: clusterID + "-" + o.conf.BcsNamespace + "-" + ist.Platform + "-" + ist.Group, DiskLeft: totalIst, MemLeft: totalIst * ist.MemPerInstance, CPULeft: totalIst * ist.CPUPerInstance, @@ -399,12 +401,14 @@ func (o *operator) getServerStatus(clusterID, namespace, name string) (*op.Servi info := &op.ServiceInfo{} if err := o.getDeployments(clusterID, namespace, name, info); err != nil { - blog.Errorf("k8s-operator: get server status, get deployments failed: %v", err) + blog.Errorf("k8s-operator: get server status, get deployments clusterID(%s) namespace(%s) failed: %v", + clusterID, namespace, err) return nil, err } if err := o.getPods(clusterID, namespace, name, info); err != nil { - blog.Errorf("k8s-operator: get server status, get pods failed: %v", err) + blog.Errorf("k8s-operator: get server status, get pods clusterID(%s) namespace(%s) failed: %v", + clusterID, namespace, err) return nil, err } @@ -712,10 +716,12 @@ func (o *operator) getClientSetFromCache(clusterID string) (*clusterClientSet, b func (o *operator) generateClient(clusterID string) (*clusterClientSet, error) { address := o.conf.BcsAPIPool.GetAddress() + var host string if o.conf.EnableBCSApiGw { - EnableBCSApiGw = "1" + host = fmt.Sprintf(bcsAPIGWK8SBaseURI, address, clusterID) + } else { + host = fmt.Sprintf(bcsAPIK8SBaseURI, address, clusterID) } - host := fmt.Sprintf(getBcsK8SBaseURL(), address, clusterID) blog.Infof("k8s-operator: try generate client with host(%s) token(%s)", host, o.conf.BcsAPIToken) // get client set by real api-server address diff --git a/src/backend/booster/server/pkg/resource/crm/operator/mesos/operator.go b/src/backend/booster/server/pkg/resource/crm/operator/mesos/operator.go index 52f7a1815c1..3731380dbc2 100644 --- a/src/backend/booster/server/pkg/resource/crm/operator/mesos/operator.go +++ b/src/backend/booster/server/pkg/resource/crm/operator/mesos/operator.go @@ -153,12 +153,14 @@ func (o *operator) getServerStatus(clusterID, namespace, name string) (*op.Servi info := &op.ServiceInfo{} if err := o.getApplication(clusterID, namespace, name, info); err != nil { - blog.Errorf("get server status, get application failed: %v", err) + blog.Errorf("get server status, clusterId(%s), ns(%s), name(%s) get application failed: %v", + clusterID, namespace, name, err) return nil, err } if err := o.getTaskGroup(clusterID, namespace, name, info); err != nil { - blog.Errorf("get server status, get taskGroup failed: %v", err) + blog.Errorf("get server status, clusterId(%s), ns(%s), name(%s) get taskGroup failed: %v", + clusterID, namespace, name, err) return nil, err } diff --git a/src/backend/booster/server/pkg/resource/direct/agent/pkg/manager/manager.go b/src/backend/booster/server/pkg/resource/direct/agent/pkg/manager/manager.go index 554e46a6b2e..cce8d33d0e1 100644 --- a/src/backend/booster/server/pkg/resource/direct/agent/pkg/manager/manager.go +++ b/src/backend/booster/server/pkg/resource/direct/agent/pkg/manager/manager.go @@ -583,15 +583,17 @@ func (o *processManager) startCommand(dir, cmdPath, processName string, params [ index++ } - // to ensure the process has running - processID := strconv.Itoa(cmd.Process.Pid) - existed := o.processExistedByNameAndPid(processName, processID) - if existed { - return cmd.Process.Pid, nil - } + // // to ensure the process has running + // processID := strconv.Itoa(cmd.Process.Pid) + // existed := o.processExistedByNameAndPid(processName, processID) + // if existed { + // return cmd.Process.Pid, nil + // } - err = fmt.Errorf("not found running process for[%s %s]", processName, processID) - blog.Infof("%v", err) + // err = fmt.Errorf("not found running process for[%s %s]", processName, processID) + // blog.Infof("%v", err) + // return 0, err + // do not check process name now, for it does not exist for *.bat return 0, err } return 0, nil diff --git a/src/backend/booster/server/pkg/server.go b/src/backend/booster/server/pkg/server.go index d139e8b01fc..6802628e8df 100644 --- a/src/backend/booster/server/pkg/server.go +++ b/src/backend/booster/server/pkg/server.go @@ -421,7 +421,7 @@ func (s *Server) initK8sResourceManagers() (k8sRm crm.ResourceManager, curQueueMap := make(map[string]bool) k8sRmList = make(map[string]crm.ResourceManager) k8sconfList := s.conf.K8sResourceConfigList - if k8sconfList.K8sClusterList != nil { + if k8sconfList.Enable && k8sconfList.K8sClusterList != nil { for key, confItem := range k8sconfList.K8sClusterList { if confItem.MySQLStorage == "" { confItem.MySQLStorage = k8sconfList.MySQLStorage @@ -452,13 +452,14 @@ func (s *Server) initK8sResourceManagers() (k8sRm crm.ResourceManager, } k8sConf := s.conf.K8sContainerResourceConfig - for queueName, istItem := range k8sQueueIstList { - if !curQueueMap[queueName] { - initInstanceType(&k8sConf, istItem) + if k8sConf.Enable { + for queueName, istItem := range k8sQueueIstList { + if !curQueueMap[queueName] { + initInstanceType(&k8sConf, istItem) + } } + k8sRm, err = s.initContainerResourceManager(&k8sConf, s.rd) } - - k8sRm, err = s.initContainerResourceManager(&k8sConf, s.rd) return } diff --git a/src/backend/ci/build.gradle.kts b/src/backend/ci/build.gradle.kts index 87dace9e1a9..03d10d062b2 100644 --- a/src/backend/ci/build.gradle.kts +++ b/src/backend/ci/build.gradle.kts @@ -14,6 +14,15 @@ allprojects { version = (System.getProperty("ci_version") ?: "1.9.0") + if (System.getProperty("snapshot") == "true") "-SNAPSHOT" else "-RELEASE" + // Docker镜像构建 + if (name.startsWith("boot-") && System.getProperty("devops.assemblyMode") == "KUBERNETES") { + pluginManager.apply("task-docker-build") + } + + // TODO bkrepo依赖到 , 后续加到framework后可以删掉 + repositories { + maven(url = "https://repo.spring.io/milestone") + } // 版本管理 dependencyManagement { setApplyMavenExclusions(false) @@ -100,9 +109,15 @@ allprojects { } dependency("com.perforce:p4java:${Versions.p4}") dependency("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${Versions.JacksonDatatypeJsr}") + dependency("io.mockk:mockk:${Versions.mockk}") dependencySet("io.github.resilience4j:${Versions.Resilience4j}") { entry("resilience4j-circuitbreaker") } + // TODO 等后面spring cloud版本升级上来就可以去掉 + dependency( + "org.springframework.cloud:spring-cloud-kubernetes-client-discovery:" + + "${Versions.KubernetesDiscovery}" + ) } } @@ -116,10 +131,11 @@ allprojects { it.exclude("javax.ws.rs", "jsr311-api") it.exclude("dom4j", "dom4j") it.exclude("com.flipkart.zjsonpatch", "zjsonpatch") - it.exclude("com.zaxxer","HikariCP-java7") + it.exclude("com.zaxxer", "HikariCP-java7") + it.exclude("com.tencent.devops", "devops-boot-starter-plugin") } - // 兼容dom4j 的 bug : https://github.com/gradle/gradle/issues/13656 dependencies { + // 兼容dom4j 的 bug : https://github.com/gradle/gradle/issues/13656 components { withModule("org.dom4j:dom4j") { allVariants { withDependencies { clear() } } diff --git a/src/backend/ci/buildSrc/build.gradle.kts b/src/backend/ci/buildSrc/build.gradle.kts index 00e610fe4b4..6ed4e26c304 100644 --- a/src/backend/ci/buildSrc/build.gradle.kts +++ b/src/backend/ci/buildSrc/build.gradle.kts @@ -5,7 +5,6 @@ plugins { // 插件使用仓库 repositories { - mavenLocal() if (System.getenv("GITHUB_WORKFLOW") == null) { // 普通环境 maven(url = "https://mirrors.tencent.com/nexus/repository/maven-public") maven(url = "https://mirrors.tencent.com/nexus/repository/gradle-plugins/") @@ -13,6 +12,7 @@ repositories { mavenCentral() gradlePluginPortal() } + mavenLocal() } // 依赖插件 @@ -21,4 +21,5 @@ dependencies { implementation("com.github.jengelman.gradle.plugins:shadow:6.1.0") implementation("org.apache.logging.log4j:log4j-core:2.17.1") implementation("org.owasp:dependency-check-gradle:7.1.0.1") + implementation("com.google.cloud.tools:jib-gradle-plugin:3.3.1") } diff --git a/src/backend/ci/buildSrc/src/main/kotlin/constants/Versions.kt b/src/backend/ci/buildSrc/src/main/kotlin/constants/Versions.kt index 3f814faca19..830f07abe1e 100644 --- a/src/backend/ci/buildSrc/src/main/kotlin/constants/Versions.kt +++ b/src/backend/ci/buildSrc/src/main/kotlin/constants/Versions.kt @@ -24,7 +24,7 @@ object Versions { const val DockerJava = "3.2.5" const val DdPlist = "1.23" const val ApkParser = "2.5.3" - const val TencentBkRepo = "1.0.0" + const val TencentBkRepo = "1.0.1" const val Ant = "1.10.5" const val Cglib = "2.2.2" const val Sigar = "1.6.4" @@ -44,6 +44,8 @@ object Versions { const val Pulsar = "2.7.2" const val JacksonDatatypeJsr = "2.11.4" const val reflections = "0.10.2" + const val mockk = "1.12.2" const val Resilience4j = "1.7.1" const val jjwt = "0.11.5" + const val KubernetesDiscovery = "2.0.6" } diff --git a/src/backend/ci/buildSrc/src/main/kotlin/plugins/task-docker-build.gradle.kts b/src/backend/ci/buildSrc/src/main/kotlin/plugins/task-docker-build.gradle.kts new file mode 100644 index 00000000000..d4d31b66046 --- /dev/null +++ b/src/backend/ci/buildSrc/src/main/kotlin/plugins/task-docker-build.gradle.kts @@ -0,0 +1,104 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +plugins { + id("com.google.cloud.tools.jib") +} + +val toImageTemplate = System.getProperty("to.image.template") +val toImageTag = System.getProperty("to.image.tag") +var toImage = System.getProperty("jib.to.image") + +if (toImage.isNullOrBlank() || (toImageTemplate.isNullOrBlank() && toImageTag.isNullOrBlank())) { + val service = name.replace("boot-", "").replace("-tencent", "") + if (toImage.isNullOrBlank() && !toImageTemplate.isNullOrBlank()) { + // 替换掉模板的__service__和__tag__ + toImage = toImageTemplate.replace("__service__", service).replace("__tag__", toImageTag) + } + + val springProfiles = System.getProperty("spring.profiles") + val serviceNamespace = System.getProperty("service.namespace") + val configNamespace = System.getProperty("config.namespace") + val jvmFlagList = System.getProperty("jvmFlags.file")?.let { + File(it).readLines().map { + it.replace("__service__", service) + .replace("__namespace__", serviceNamespace) + } + } ?: emptyList() + + val finalJvmFlags = mutableListOf( + "-server", + "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8080", + "-Xloggc:/data/workspace/$service/jvm/gc-%t.log", + "-XX:+PrintTenuringDistribution", + "-XX:+PrintGCDetails", + "-XX:+PrintGCDateStamps", + "-XX:MaxGCPauseMillis=200", + "-XX:+UseG1GC", + "-XX:NativeMemoryTracking=summary", + "-XX:+HeapDumpOnOutOfMemoryError", + "-XX:HeapDumpPath=/data/workspace/$service/jvm/oom.hprof", + "-XX:ErrorFile=/data/workspace/$service/jvm/error_sys.log", + "-XX:+UseContainerSupport", + "-XX:InitialRAMPercentage=70.0", + "-XX:MaxRAMPercentage=70.0", + "-XX:MaxRAMPercentage=70.0", + "-XX:-UseAdaptiveSizePolicy", + "-Dspring.profiles.active=$springProfiles", + "-Dspring.jmx.enabled=true", + "-Dservice.log.dir=/data/workspace/$service/logs/", + "-Dsun.jnu.encoding=UTF-8", + "-Dfile.encoding=UTF-8", + "-Dspring.main.allow-bean-definition-overriding=true", + "-Djasypt.encryptor.bootstrap=false", + "-Dspring.cloud.config.fail-fast=true", + "-Dspring.main.allow-circular-references=true", + "-Dspring.cloud.kubernetes.config.sources[0].name=config-bk-ci-common", + "-Dspring.cloud.kubernetes.config.sources[1].name=config-bk-ci-$service", + "-Dspring.cloud.kubernetes.config.namespace=$configNamespace", + "-Dspring.cloud.kubernetes.discovery.all-namespaces=true", + "-Dio.undertow.legacy.cookie.ALLOW_HTTP_SEPARATORS_IN_V0=true", + "-Dserver.port=80" + ) + finalJvmFlags.addAll(jvmFlagList) + + jib { + // 环境变量 + container { + environment = hashMapOf("INNER_NAME" to "bk-ci") + } + // 缓存位置 + System.setProperty("jib.applicationCache", "~/.gradle/jib-cache") + // 启动参数 + container { + jvmFlags = finalJvmFlags + } + // 目标镜像 + to { + image = toImage + } + } +} diff --git a/src/backend/ci/core/artifactory/api-artifactory-store/src/main/kotlin/com/tencent/devops/artifactory/api/ServiceArchiveAtomFileResource.kt b/src/backend/ci/core/artifactory/api-artifactory-store/src/main/kotlin/com/tencent/devops/artifactory/api/ServiceArchiveAtomFileResource.kt new file mode 100644 index 00000000000..f8dbe9dccfe --- /dev/null +++ b/src/backend/ci/core/artifactory/api-artifactory-store/src/main/kotlin/com/tencent/devops/artifactory/api/ServiceArchiveAtomFileResource.kt @@ -0,0 +1,113 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.artifactory.api + +import com.tencent.devops.artifactory.pojo.ArchiveAtomResponse +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.annotation.BkField +import com.tencent.devops.store.pojo.common.enums.ReleaseTypeEnum +import io.swagger.annotations.Api +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import org.glassfish.jersey.media.multipart.FormDataContentDisposition +import org.glassfish.jersey.media.multipart.FormDataParam +import java.io.InputStream +import javax.ws.rs.Consumes +import javax.ws.rs.POST +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.QueryParam +import javax.ws.rs.core.MediaType + +@Api(tags = ["SERVICE_ARTIFACTORY"], description = "仓库-插件") +@Path("/service/artifactories/") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface ServiceArchiveAtomFileResource { + + @ApiOperation("归档插件包资源") + @POST + @Path("/archiveAtom") + @Consumes(MediaType.MULTIPART_FORM_DATA) + fun archiveAtomFile( + @ApiParam("userId", required = true) + @QueryParam("userId") + userId: String, + @ApiParam("项目编码", required = true) + @QueryParam("projectCode") + projectCode: String, + @ApiParam("插件ID", required = true) + @QueryParam("atomId") + atomId: String, + @ApiParam("插件代码", required = true) + @QueryParam("atomCode") + atomCode: String, + @ApiParam("插件版本号", required = true) + @QueryParam("version") + version: String, + @ApiParam("发布类型", required = true) + @QueryParam("releaseType") + releaseType: ReleaseTypeEnum, + @ApiParam("文件", required = true) + @FormDataParam("file") + inputStream: InputStream, + @FormDataParam("file") + disposition: FormDataContentDisposition, + @ApiParam("支持的操作系统", required = true) + @QueryParam("os") + os: String + ): Result + + @ApiOperation("上传插件资源文件到指定自定义仓库路径") + @POST + @Path("/file/uploadToPath") + @Consumes(MediaType.MULTIPART_FORM_DATA) + fun uploadToPath( + @ApiParam("userId", required = true) + @QueryParam("userId") + @BkField(required = true) + userId: String, + @ApiParam("项目代码", required = true) + @QueryParam("projectId") + @BkField(required = true) + projectId: String, + @ApiParam("文件路径", required = true) + @QueryParam("path") + @BkField(required = true) + path: String, + @ApiParam("文件类型", required = true) + @QueryParam("fileType") + @BkField(required = true) + fileType: String, + @ApiParam("文件", required = true) + @FormDataParam("file") + inputStream: InputStream, + @FormDataParam("file") + disposition: FormDataContentDisposition + ): Result +} diff --git a/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileDetail.kt b/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileDetail.kt index c211d8cdbb5..f7066a824d9 100644 --- a/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileDetail.kt +++ b/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileDetail.kt @@ -27,6 +27,7 @@ package com.tencent.devops.artifactory.pojo +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import io.swagger.annotations.ApiModel import io.swagger.annotations.ApiModelProperty @@ -49,5 +50,7 @@ data class FileDetail( @ApiModelProperty("文件摘要", required = true) val checksums: FileChecksums, @ApiModelProperty("meta数据", required = true) - val meta: Map + val meta: Map, + @ApiModelProperty("nodeMetadata数据", required = true) + val nodeMetadata: List = emptyList() ) diff --git a/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileDetailForApp.kt b/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileDetailForApp.kt index fe041014f77..47fa11e197e 100644 --- a/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileDetailForApp.kt +++ b/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileDetailForApp.kt @@ -27,6 +27,7 @@ package com.tencent.devops.artifactory.pojo +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.devops.artifactory.pojo.enums.ArtifactoryType import io.swagger.annotations.ApiModel import io.swagger.annotations.ApiModelProperty @@ -64,5 +65,7 @@ data class FileDetailForApp( @ApiModelProperty("md5", required = true) val md5: String, @ApiModelProperty("构建号", required = true) - val buildNum: Int + val buildNum: Int, + @ApiModelProperty("nodeMetadata数据", required = true) + val nodeMetadata: List = emptyList() ) diff --git a/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileInfo.kt b/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileInfo.kt index 02c2ab1d582..6851add0f49 100644 --- a/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileInfo.kt +++ b/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/FileInfo.kt @@ -58,7 +58,9 @@ data class FileInfo( @ApiModelProperty("下载链接", required = false) var downloadUrl: String? = null, @ApiModelProperty("MD5", required = false) - var md5: String? = null + var md5: String? = null, + @ApiModelProperty("docker registry", required = false) + var registry: String? = null ) : Comparable { constructor( name: String, diff --git a/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/enums/FileTypeEnum.kt b/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/enums/FileTypeEnum.kt index ab16665ceb7..b0b7f1bfac5 100644 --- a/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/enums/FileTypeEnum.kt +++ b/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/pojo/enums/FileTypeEnum.kt @@ -32,7 +32,6 @@ enum class FileTypeEnum(val fileType: String) { BK_CUSTOM("bk-custom"), // 指定了自定义路径的归档类型，会覆盖 BK_REPORT("bk-report"), // 报告产出物 BK_PLUGIN_FE("bk-plugin-fe"); // 插件自定义UI前端文件 - fun toArtifactoryType(): ArtifactoryType { return when (this) { BK_ARCHIVE -> ArtifactoryType.PIPELINE diff --git a/src/backend/ci/core/artifactory/biz-artifactory-store/src/main/kotlin/com/tencent/devops/artifactory/resources/ServiceArchiveAtomFileResourceImpl.kt b/src/backend/ci/core/artifactory/biz-artifactory-store/src/main/kotlin/com/tencent/devops/artifactory/resources/ServiceArchiveAtomFileResourceImpl.kt new file mode 100644 index 00000000000..69deffd5078 --- /dev/null +++ b/src/backend/ci/core/artifactory/biz-artifactory-store/src/main/kotlin/com/tencent/devops/artifactory/resources/ServiceArchiveAtomFileResourceImpl.kt @@ -0,0 +1,95 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.artifactory.resources + +import com.tencent.devops.artifactory.api.ServiceArchiveAtomFileResource +import com.tencent.devops.artifactory.pojo.ArchiveAtomRequest +import com.tencent.devops.artifactory.pojo.ArchiveAtomResponse +import com.tencent.devops.artifactory.pojo.enums.FileChannelTypeEnum +import com.tencent.devops.artifactory.pojo.enums.FileTypeEnum +import com.tencent.devops.artifactory.service.ArchiveAtomService +import com.tencent.devops.artifactory.service.ArchiveFileService +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.store.pojo.common.enums.ReleaseTypeEnum +import org.glassfish.jersey.media.multipart.FormDataContentDisposition +import org.springframework.beans.factory.annotation.Autowired +import java.io.InputStream + +@RestResource +class ServiceArchiveAtomFileResourceImpl @Autowired constructor( + private val archiveAtomService: ArchiveAtomService, + private val archiveFileService: ArchiveFileService +) : ServiceArchiveAtomFileResource { + + override fun archiveAtomFile( + userId: String, + projectCode: String, + atomId: String, + atomCode: String, + version: String, + releaseType: ReleaseTypeEnum, + inputStream: InputStream, + disposition: FormDataContentDisposition, + os: String + ): Result { + return archiveAtomService.archiveAtom( + userId = userId, + inputStream = inputStream, + disposition = disposition, + atomId = atomId, + archiveAtomRequest = ArchiveAtomRequest( + projectCode = projectCode, + atomCode = atomCode, + version = version, + releaseType = releaseType, + os = os + ) + ) + } + + override fun uploadToPath( + userId: String, + projectId: String, + path: String, + fileType: String, + inputStream: InputStream, + disposition: FormDataContentDisposition + ): Result { + val url = archiveFileService.uploadFile( + userId = userId, + inputStream = inputStream, + disposition = disposition, + projectId = projectId, + filePath = path, + fileType = FileTypeEnum.valueOf(fileType), + fileChannelType = FileChannelTypeEnum.WEB_SHOW + ) + return Result(url) + } +} diff --git a/src/backend/ci/core/artifactory/biz-artifactory-store/src/main/kotlin/com/tencent/devops/artifactory/service/impl/ArchiveAtomToBkRepoServiceImpl.kt b/src/backend/ci/core/artifactory/biz-artifactory-store/src/main/kotlin/com/tencent/devops/artifactory/service/impl/ArchiveAtomToBkRepoServiceImpl.kt index e807689ed19..886b1188629 100644 --- a/src/backend/ci/core/artifactory/biz-artifactory-store/src/main/kotlin/com/tencent/devops/artifactory/service/impl/ArchiveAtomToBkRepoServiceImpl.kt +++ b/src/backend/ci/core/artifactory/biz-artifactory-store/src/main/kotlin/com/tencent/devops/artifactory/service/impl/ArchiveAtomToBkRepoServiceImpl.kt @@ -43,25 +43,42 @@ class ArchiveAtomToBkRepoServiceImpl : ArchiveAtomServiceImpl() { val atomArchivePath = buildAtomArchivePath(projectCode, atomCode, version) val frontendDir = buildAtomFrontendPath(atomCode, version) logger.info("atom plugin: $atomArchivePath, $frontendDir") - File(atomArchivePath).walk().filter { it.path != atomArchivePath }.forEach { - val path = it.path.removePrefix("${getAtomArchiveBasePath()}/$BK_CI_ATOM_DIR") - bkRepoClient.uploadLocalFile( - userId = BKREPO_DEFAULT_USER, - projectId = BKREPO_STORE_PROJECT_ID, - repoName = REPO_NAME_PLUGIN, - path = path, - file = it - ) - } - File(frontendDir).walk().filter { it.path != frontendDir }.forEach { - val path = it.path.removePrefix("${getAtomArchiveBasePath()}/$STATIC/$BK_CI_PLUGIN_FE_DIR") - bkRepoClient.uploadLocalFile( - userId = BKREPO_DEFAULT_USER, - projectId = BKREPO_STORE_PROJECT_ID, - repoName = REPO_NAME_STATIC, - path = path, - file = it - ) + + directoryIteration( + directoryFile = File(atomArchivePath), + prefix = "${getAtomArchiveBasePath()}/$BK_CI_ATOM_DIR", + directoryPath = atomArchivePath, + repoName = REPO_NAME_PLUGIN + ) + directoryIteration( + directoryFile = File(frontendDir), + prefix = "${getAtomArchiveBasePath()}/$STATIC/$BK_CI_PLUGIN_FE_DIR", + directoryPath = frontendDir, + repoName = REPO_NAME_STATIC + ) + } + + private fun directoryIteration(directoryFile: File, prefix: String, directoryPath: String, repoName: String) { + directoryFile.walk().filter { it.path != directoryPath }.forEach { + if (it.isDirectory) { + directoryIteration( + directoryFile = it, + prefix = prefix, + directoryPath = it.path, + repoName = repoName + ) + } else { + val path = it.path.removePrefix(prefix) + logger.debug("uploadLocalFile fileName=${it.name}|path=$path") + + bkRepoClient.uploadLocalFile( + userId = BKREPO_DEFAULT_USER, + projectId = BKREPO_STORE_PROJECT_ID, + repoName = repoName, + path = path, + file = it + ) + } } } diff --git a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/BkRepoArchiveFileServiceImpl.kt b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/BkRepoArchiveFileServiceImpl.kt index 618d647a79b..71b485b71d3 100644 --- a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/BkRepoArchiveFileServiceImpl.kt +++ b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/BkRepoArchiveFileServiceImpl.kt @@ -41,6 +41,8 @@ import com.tencent.devops.artifactory.util.BkRepoUtils.BKREPO_DEFAULT_USER import com.tencent.devops.artifactory.util.BkRepoUtils.BKREPO_DEVOPS_PROJECT_ID import com.tencent.devops.artifactory.util.BkRepoUtils.BKREPO_STORE_PROJECT_ID import com.tencent.devops.artifactory.util.BkRepoUtils.REPO_NAME_CUSTOM +import com.tencent.devops.artifactory.util.BkRepoUtils.REPO_NAME_IMAGE +import com.tencent.devops.artifactory.util.BkRepoUtils.REPO_NAME_PIPELINE import com.tencent.devops.artifactory.util.BkRepoUtils.REPO_NAME_REPORT import com.tencent.devops.artifactory.util.BkRepoUtils.REPO_NAME_STATIC import com.tencent.devops.artifactory.util.BkRepoUtils.parseArtifactoryType @@ -59,6 +61,7 @@ import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.common.auth.api.AuthResourceType import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.stereotype.Service import org.springframework.web.context.request.RequestContextHolder @@ -79,6 +82,9 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( private val bkRepoClient: BkRepoClient ) : ArchiveFileServiceImpl() { + @Value("\${bkrepo.dockerRegistry:#{null}}") + private val dockerRegistry: String? = null + override fun show(userId: String, projectId: String, artifactoryType: ArtifactoryType, path: String): FileDetail { val nodeDetail = bkRepoClient.getFileDetail(userId = userId, projectId = projectId, @@ -99,7 +105,8 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( fileChannelType: FileChannelTypeEnum, logo: Boolean? ): String { - val destPath = filePath ?: DefaultPathUtils.randomFileName() + val pathSplit = file.name.split('.') + val destPath = filePath ?: DefaultPathUtils.randomFileName(pathSplit[pathSplit.size - 1]) val metadata = mutableMapOf() metadata["shaContent"] = file.inputStream().use { ShaUtils.sha1InputStream(it) } props?.forEach { @@ -220,7 +227,7 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( val nodeList = bkRepoClient.queryByNameAndMetadata( userId = userId, projectId = projectId, - repoNames = listOf(BkRepoUtils.REPO_NAME_PIPELINE, REPO_NAME_CUSTOM), + repoNames = listOf(REPO_NAME_PIPELINE, REPO_NAME_CUSTOM, REPO_NAME_IMAGE), fileNames = listOf(), metadata = searchProps.props, page = page ?: 1, @@ -256,7 +263,8 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( .timestamp(), folder = false, artifactoryType = ArtifactoryType.IMAGE, - properties = metadata.map { m -> Property(m["key"].toString(), m["value"].toString()) } + properties = metadata.map { m -> Property(m["key"].toString(), m["value"].toString()) }, + registry = dockerRegistry ) } } else { diff --git a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/DiskArchiveFileServiceImpl.kt b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/DiskArchiveFileServiceImpl.kt index c5ad8a32566..01f5886378d 100644 --- a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/DiskArchiveFileServiceImpl.kt +++ b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/DiskArchiveFileServiceImpl.kt @@ -359,8 +359,9 @@ class DiskArchiveFileServiceImpl : ArchiveFileServiceImpl() { logger.info("uploadFile|filePath=$filePath|fileName=$fileName|props=$props") val uploadFileName = fileName ?: file.name val fileTypeStr = fileType?.fileType ?: "file" + val fileTypeName = file.name.substring(file.name.indexOf(".") + 1) val destPath = if (null == filePath) { - "${getBasePath()}$fileSeparator$fileTypeStr$fileSeparator$${DefaultPathUtils.randomFileName()}" + "${getBasePath()}$fileSeparator$fileTypeStr$fileSeparator$${DefaultPathUtils.randomFileName(fileTypeName)}" } else { // #5176 修正未对上传类型来决定存放路径的问题，统一在此生成归档路径，而不是由外部指定会存在内部路径泄露风险 if (fileType != null && !projectId.isNullOrBlank()) { diff --git a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceProjectAuthResource.kt b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceProjectAuthResource.kt index 880fa09008d..aa205fd0e59 100644 --- a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceProjectAuthResource.kt +++ b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceProjectAuthResource.kt @@ -29,6 +29,7 @@ package com.tencent.devops.auth.api.service import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_BK_TOKEN import com.tencent.devops.common.api.auth.AUTH_HEADER_GIT_TYPE +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.pojo.BKAuthProjectRolesResources import com.tencent.devops.common.auth.api.pojo.BkAuthGroup @@ -133,6 +134,21 @@ interface ServiceProjectAuthResource { projectCode: String ): Result + @GET + @Path("/projectIds/{projectId}/checkManager") + @ApiOperation("判断是否是项目管理员或CI管理员") + fun checkManager( + @HeaderParam(AUTH_HEADER_DEVOPS_BK_TOKEN) + @ApiParam("认证token", required = true) + token: String, + @ApiParam(name = "用户名", required = true) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + @ApiParam("项目Id", required = true) + projectId: String + ): Result + @POST @Path("/{projectCode}/createUser") @ApiOperation("添加用户到指定项目指定分组") diff --git a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/user/UserProjectMemberResource.kt b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/user/UserProjectMemberResource.kt index 2285b310f3b..eb820e05d2d 100644 --- a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/user/UserProjectMemberResource.kt +++ b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/user/UserProjectMemberResource.kt @@ -144,4 +144,16 @@ interface UserProjectMemberResource { @ApiParam(name = "待搜用户", required = true) searchUserId: String ): Result?> + + @GET + @Path("/projectIds/{projectId}/checkManager") + @ApiOperation("判断是否是项目管理员或CI管理员") + fun checkManager( + @ApiParam(name = "用户名", required = true) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + @ApiParam("项目Id", required = true) + projectId: String + ): Result } diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserProjectMemberResourceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserProjectMemberResourceImpl.kt index 376b4158734..c14de3fe690 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserProjectMemberResourceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserProjectMemberResourceImpl.kt @@ -34,14 +34,17 @@ import com.tencent.bk.sdk.iam.dto.manager.vo.ManagerGroupMemberVo import com.tencent.devops.auth.api.user.UserProjectMemberResource import com.tencent.devops.auth.pojo.dto.RoleMemberDTO import com.tencent.devops.auth.pojo.vo.ProjectMembersVO +import com.tencent.devops.auth.service.iam.PermissionProjectService import com.tencent.devops.auth.service.iam.PermissionRoleMemberService import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import com.tencent.devops.common.web.RestResource import org.springframework.beans.factory.annotation.Autowired @RestResource class UserProjectMemberResourceImpl @Autowired constructor( - val permissionRoleMemberService: PermissionRoleMemberService + val permissionRoleMemberService: PermissionRoleMemberService, + val permissionProjectService: PermissionProjectService ) : UserProjectMemberResource { override fun createRoleMember( userId: String, @@ -73,7 +76,8 @@ class UserProjectMemberResourceImpl @Autowired constructor( roleId = roleId, page = page, pageSize = pageSize - )) + ) + ) } override fun getProjectAllMember(projectId: Int, page: Int?, pageSize: Int?): Result { @@ -88,14 +92,16 @@ class UserProjectMemberResourceImpl @Autowired constructor( members: String, type: ManagerScopesEnum ): Result { - Result(permissionRoleMemberService.deleteRoleMember( - userId = userId, - projectId = projectId, - roleId = roleId, - id = members, - type = type, - managerGroup = managerGroup - )) + Result( + permissionRoleMemberService.deleteRoleMember( + userId = userId, + projectId = projectId, + roleId = roleId, + id = members, + type = type, + managerGroup = managerGroup + ) + ) return Result(true) } @@ -106,4 +112,10 @@ class UserProjectMemberResourceImpl @Autowired constructor( ): Result?> { return Result(permissionRoleMemberService.getUserGroups(projectId, searchUserId)) } + + override fun checkManager(userId: String, projectId: String): Result { + val result = permissionProjectService.checkProjectManager(userId, projectId) || + permissionProjectService.isProjectUser(userId, projectId, BkAuthGroup.CI_MANAGER) + return Result(result) + } } diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceProjectAuthResourceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceProjectAuthResourceImpl.kt index f9a1ec4ffef..39611d3fa1d 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceProjectAuthResourceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceProjectAuthResourceImpl.kt @@ -85,6 +85,12 @@ class ServiceProjectAuthResourceImpl @Autowired constructor( ) } + override fun checkManager(token: String, userId: String, projectId: String): Result { + val result = permissionProjectService.checkProjectManager(userId, projectId) || + permissionProjectService.isProjectUser(userId, projectId, BkAuthGroup.CI_MANAGER) + return Result(result) + } + override fun checkProjectManager( token: String, type: String?, diff --git a/src/backend/ci/core/buildless/biz-buildless/build.gradle.kts b/src/backend/ci/core/buildless/biz-buildless/build.gradle.kts index 2f45271316e..4cacc67a73d 100644 --- a/src/backend/ci/core/buildless/biz-buildless/build.gradle.kts +++ b/src/backend/ci/core/buildless/biz-buildless/build.gradle.kts @@ -30,9 +30,7 @@ dependencies { api(project(":core:common:common-service")) api(project(":core:common:common-web")) - api(project(":core:common:common-client")) api(project(":core:common:common-redis")) - api(project(":core:common:common-db")) api(project(":core:log:api-log")) api("com.github.docker-java:docker-java") diff --git a/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/client/DispatchClient.kt b/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/client/DispatchClient.kt index 3a5a7e9656b..8a32d8955b3 100644 --- a/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/client/DispatchClient.kt +++ b/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/client/DispatchClient.kt @@ -27,36 +27,68 @@ package com.tencent.devops.buildless.client +import com.tencent.devops.buildless.config.BuildLessConfig import com.tencent.devops.buildless.pojo.BuildLessTask import com.tencent.devops.buildless.utils.CommonUtils import com.tencent.devops.buildless.utils.SystemInfoUtil -import com.tencent.devops.common.client.Client +import com.tencent.devops.common.api.auth.AUTH_HEADER_GATEWAY_TAG +import com.tencent.devops.common.api.exception.TaskExecuteException +import com.tencent.devops.common.api.pojo.ErrorCode +import com.tencent.devops.common.api.pojo.ErrorType +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.api.util.OkhttpUtils import com.tencent.devops.common.service.BkTag import com.tencent.devops.common.service.config.CommonConfig -import com.tencent.devops.dispatch.docker.api.service.ServiceDockerHostResource import com.tencent.devops.dispatch.docker.pojo.DockerIpInfoVO import com.tencent.devops.dispatch.docker.pojo.enums.DockerHostClusterType +import okhttp3.Headers +import okhttp3.MediaType +import okhttp3.Request +import okhttp3.RequestBody import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @Component class DispatchClient @Autowired constructor( - private val client: Client, + private val buildLessConfig: BuildLessConfig, private val commonConfig: CommonConfig, private val bkTag: BkTag ) { fun updateContainerId(buildLessTask: BuildLessTask, containerId: String) { - client.get(ServiceDockerHostResource::class).updateContainerId( - buildId = buildLessTask.buildId, - vmSeqId = buildLessTask.vmSeqId, - containerId = containerId - ) + val path = "/ms/dispatch-docker/api/service/dockerhost/builds/${buildLessTask.buildId}/vmseqs" + + "/${buildLessTask.vmSeqId}?containerId=$containerId" + + try { + val url = buildUrl(path) + val request = Request + .Builder() + .url(url) + .headers(Headers.of(makeHeaders())) + .put(RequestBody.create( + MediaType.parse("application/json; charset=utf-8"), + "") + ) + .build() + + OkhttpUtils.doHttp(request).use { response -> + val responseContent = response.body()!!.string() + if (!response.isSuccessful) { + logger.error("Update containerId $path fail. $responseContent") + throw TaskExecuteException( + errorCode = ErrorCode.SYSTEM_WORKER_INITIALIZATION_ERROR, + errorType = ErrorType.SYSTEM, + errorMsg = "Update containerId $path fail") + } + } + } catch (e: Exception) { + logger.error("Update containerId failed. errorInfo: ${e.message}") + } } fun refreshStatus(containerRunningsCount: Int) { val dockerIp = CommonUtils.getHostIp() - + val path = "/ms/dispatch-docker/api/service/dockerhost/dockerIp/$dockerIp/refresh" // 节点状态默认正常 var enable = true @@ -84,10 +116,29 @@ class DispatchClient @Autowired constructor( ) try { - client.get(ServiceDockerHostResource::class).refresh( - dockerIp = dockerIp, - dockerIpInfoVO = dockerIpInfoVO - ) + val url = buildUrl(path) + val request = Request + .Builder() + .url(url) + .headers(Headers.of(makeHeaders())) + .post(RequestBody.create( + MediaType.parse("application/json; charset=utf-8"), + JsonUtil.toJson(dockerIpInfoVO)) + ) + .build() + + logger.info("Start refresh buildLess status $url") + OkhttpUtils.doHttp(request).use { response -> + val responseContent = response.body()!!.string() + if (!response.isSuccessful) { + logger.error("Refresh buildLess status $url fail. $responseContent") + throw TaskExecuteException( + errorCode = ErrorCode.SYSTEM_WORKER_INITIALIZATION_ERROR, + errorType = ErrorType.SYSTEM, + errorMsg = "Refresh buildLess status $url fail") + } + logger.info("End refreshDockerIpStatus.") + } } catch (e: Exception) { logger.error("Refresh buildLess status failed. errorInfo: ${e.message}") } @@ -97,6 +148,31 @@ class DispatchClient @Autowired constructor( return bkTag.getLocalTag().contains("gray") } + private fun buildUrl(path: String): String { + return if (path.startsWith("http://") || path.startsWith("https://")) { + path + } else { + fixUrl(commonConfig.devopsIdcGateway!!, path) + } + } + + private fun fixUrl(server: String, path: String): String { + return if (server.startsWith("http://") || server.startsWith("https://")) { + "$server/${path.removePrefix("/")}" + } else { + "http://$server/${path.removePrefix("/")}" + } + } + + private fun makeHeaders(): Map { + val gatewayHeaderTag = if (buildLessConfig.gatewayHeaderTag == null) { + bkTag.getLocalTag() + } else { + buildLessConfig.gatewayHeaderTag + } + return mapOf(AUTH_HEADER_GATEWAY_TAG to gatewayHeaderTag) + } + companion object { private val logger = LoggerFactory.getLogger(DispatchClient::class.java) } diff --git a/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/config/BuildLessConfig.kt b/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/config/BuildLessConfig.kt index ce4f5d58e91..9207362ca50 100644 --- a/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/config/BuildLessConfig.kt +++ b/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/config/BuildLessConfig.kt @@ -98,4 +98,7 @@ class BuildLessConfig { @Value("\${containerPool.baseImage:blueking/bk-ci}") var containerPoolBaseImage: String = "blueking/bk-ci" // 容器池默认镜像 + + @Value("\${gatewayHeaderTag:#{null}}") + var gatewayHeaderTag: String? = null } diff --git a/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/utils/Constants.kt b/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/utils/Constants.kt index 8463047f4f4..7c7e40503ac 100644 --- a/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/utils/Constants.kt +++ b/src/backend/ci/core/buildless/biz-buildless/src/main/kotlin/com/tencent/devops/buildless/utils/Constants.kt @@ -49,6 +49,7 @@ const val WORKSPACE_ENV = "WORKSPACE" const val ENV_KEY_DISTCC = "DISTCC_HOSTS" const val ENV_KEY_PROJECT_ID = "devops_project_id" +const val ENV_KEY_BUILD_ID = "devops_build_id" const val ENV_KEY_AGENT_ID = "devops_agent_id" const val ENV_KEY_AGENT_SECRET_KEY = "devops_agent_secret_key" const val ENV_KEY_GATEWAY = "devops_gateway" diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonConstants.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonConstants.kt index 59f3c952220..96a28373c6c 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonConstants.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonConstants.kt @@ -108,6 +108,7 @@ const val SYSTEM = "system" // 系统 const val BUILD_RUNNING = "buildRunning" // 运行中 const val BUILD_QUEUE = "buildQueue" // 构建排队中 const val BUILD_REVIEWING = "buildReviewing" // 构建待审核 +const val BUILD_STAGE_SUCCESS = "buildStageSuccess" // 构建阶段性完成 const val BUILD_COMPLETED = "buildCompleted" // 运行成功 const val BUILD_CANCELED = "buildCanceled" // 构建已取消 const val BUILD_FAILED = "buildFailed" // 构建失败 diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/RepositoryTypeNew.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/RepositoryTypeNew.kt index 6003e93d4f3..2fe10833f49 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/RepositoryTypeNew.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/RepositoryTypeNew.kt @@ -31,4 +31,12 @@ enum class RepositoryTypeNew { ID, NAME, URL + ; + + companion object { + fun parseType(type: String?): RepositoryTypeNew { + if (type.isNullOrBlank()) return ID + return valueOf(type) + } + } } diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/SimpleResult.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/SimpleResult.kt index ab79398f9bb..05455b1d72e 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/SimpleResult.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/SimpleResult.kt @@ -35,5 +35,14 @@ data class SimpleResult( @ApiModelProperty("是否成功", required = true) val success: Boolean, @ApiModelProperty("错误信息", required = false) - val message: String? = null + val message: String? = null, + @ApiModelProperty("错误码信息", required = false) + val error: Error? = null +) + +@ApiModel("第三方构建信息模型-错误信息") +data class Error( + val errorType: String, + val errorMessage: String, + val errorCode: Int ) diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/AgentPropsInfo.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/AgentPropsInfo.kt index eee5473836d..6340b5b263f 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/AgentPropsInfo.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/AgentPropsInfo.kt @@ -8,5 +8,7 @@ data class AgentPropsInfo( @ApiModelProperty("agent运行系统的架构信息") val arch: String, @ApiModelProperty("jdk版本信息") - val jdkVersion: List? + val jdkVersion: List?, + @ApiModelProperty("docker init 文件升级信息") + val dockerInitFileInfo: DockerInitFileInfo? ) diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/DockerInitFileInfo.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/DockerInitFileInfo.kt new file mode 100644 index 00000000000..414686f76d2 --- /dev/null +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/DockerInitFileInfo.kt @@ -0,0 +1,12 @@ +package com.tencent.devops.common.api.pojo.agent + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("docker init 文件升级信息") +data class DockerInitFileInfo( + @ApiModelProperty("文件md5值") + val fileMd5: String, + @ApiModelProperty("目前只支持linux机器，所以其他系统不需要检查") + val needUpgrade: Boolean +) diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/NewHeartbeatInfo.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/NewHeartbeatInfo.kt index 861abbd28c0..ecb6b86fbcb 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/NewHeartbeatInfo.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/NewHeartbeatInfo.kt @@ -47,7 +47,7 @@ data class NewHeartbeatInfo( @ApiModelProperty("启动者") val startedUser: String, @ApiModelProperty("第三方构建信息列表") - var taskList: List, + var taskList: List?, @ApiModelProperty("Agent属性信息") val props: AgentPropsInfo?, @ApiModelProperty("构建机id") @@ -57,7 +57,13 @@ data class NewHeartbeatInfo( @ApiModelProperty("心跳时间戳") var heartbeatTime: Long?, @ApiModelProperty("忙碌运行中任务数量") - var busyTaskSize: Int = 0 + var busyTaskSize: Int = 0, + @ApiModelProperty("docker并行任务计数") + val dockerParallelTaskCount: Int?, + @ApiModelProperty("docker构建信息列表") + var dockerTaskList: List?, + @ApiModelProperty("忙碌运行docker中任务数量") + var dockerBusyTaskSize: Int = 0 ) { companion object { fun dummyHeartbeat(projectId: String, agentId: Long): NewHeartbeatInfo { @@ -70,10 +76,12 @@ data class NewHeartbeatInfo( agentInstallPath = "", startedUser = "", taskList = listOf(), - props = AgentPropsInfo("", null), + props = AgentPropsInfo("", null, null), agentId = agentId, projectId = projectId, - heartbeatTime = System.currentTimeMillis() + heartbeatTime = System.currentTimeMillis(), + dockerParallelTaskCount = 0, + dockerTaskList = listOf() ) } } diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/ThirdPartyBuildInfo.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/ThirdPartyBuildInfo.kt index dfdb4d12e5b..9e2ba77a57c 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/ThirdPartyBuildInfo.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/ThirdPartyBuildInfo.kt @@ -41,3 +41,13 @@ data class ThirdPartyBuildInfo( @ApiModelProperty("工作空间") val workspace: String ) + +@ApiModel("第三方构建Docker信息") +data class ThirdPartyDockerBuildInfo( + @ApiModelProperty("项目id") + val projectId: String, + @ApiModelProperty("构建id") + val buildId: String, + @ApiModelProperty("构建机编排序号") + val vmSeqId: String +) diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/UpgradeItem.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/UpgradeItem.kt index cc21ebb7614..f1e584b002e 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/UpgradeItem.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/agent/UpgradeItem.kt @@ -10,5 +10,7 @@ data class UpgradeItem( @ApiModelProperty("升级worker") val worker: Boolean, @ApiModelProperty("升级jdk") - val jdk: Boolean + val jdk: Boolean, + @ApiModelProperty("升级docker init 脚本") + val dockerInitFile: Boolean ) diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/FileUtil.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/FileUtil.kt index 36ddc20ee5a..6ae529bdce7 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/FileUtil.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/FileUtil.kt @@ -257,4 +257,18 @@ object FileUtil { } } } + + /** + * 写文件 + */ + fun outFile(path: String, name: String, context: String) { + val inPath = File(path) + if (!inPath.exists()) { + inPath.mkdirs() + } + val file = File("$path/$name") + file.bufferedWriter().use { out -> + out.write(context) + } + } } diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/Watcher.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/Watcher.kt index 602633742d8..6cab4e3b470 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/Watcher.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/Watcher.kt @@ -27,6 +27,7 @@ package com.tencent.devops.common.api.util +import org.slf4j.LoggerFactory import org.springframework.util.StopWatch /** @@ -76,4 +77,22 @@ class Watcher(id: String = "") : StopWatch(id) { } } } + + /** + * 监听action耗时 , 会忽略异常 + */ + fun safeAround(taskName: String, action: () -> Unit) { + try { + this.start(taskName) + action() + } catch (e: Exception) { + logger.warn("$id , $taskName", e) + } finally { + this.stop() + } + } + + companion object { + private val logger = LoggerFactory.getLogger(Watcher::class.java) + } } diff --git a/src/backend/ci/core/common/common-api/src/test/kotlin/com/tencent/devops/common/api/util/FileUtilTest.kt b/src/backend/ci/core/common/common-api/src/test/kotlin/com/tencent/devops/common/api/util/FileUtilTest.kt index c86e50aaeb8..b96b767d971 100644 --- a/src/backend/ci/core/common/common-api/src/test/kotlin/com/tencent/devops/common/api/util/FileUtilTest.kt +++ b/src/backend/ci/core/common/common-api/src/test/kotlin/com/tencent/devops/common/api/util/FileUtilTest.kt @@ -27,6 +27,7 @@ package com.tencent.devops.common.api.util +import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test import java.io.File @@ -41,4 +42,13 @@ class FileUtilTest { assert("bd94e431dfd6319590fe5908dd36d54a" == FileUtil.getMD5(file.readText())) assert("bd94e431dfd6319590fe5908dd36d54a" == FileUtil.getMD5(file.readBytes())) } + + @Test + fun outFile() { + FileUtil.outFile("build/", "test", "test") + val file = File("build/test") + Assertions.assertEquals(file.exists(), true) + Assertions.assertEquals(file.name, "test") + Assertions.assertEquals(file.readText(), "test") + } } diff --git a/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/pojo/BkAuthGroup.kt b/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/pojo/BkAuthGroup.kt index 2e75ff1dfb1..172e7d6b3de 100644 --- a/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/pojo/BkAuthGroup.kt +++ b/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/pojo/BkAuthGroup.kt @@ -31,13 +31,14 @@ package com.tencent.devops.common.auth.api.pojo * 项目角色组 */ enum class BkAuthGroup(val value: String) { - CIADMIN("ciAdmin"), // CI管理员 + CIADMIN("ciAdmin"), // CI管理员 TODO : 看IAM接口找不到这个标志, 用的是ci_manager MANAGER("manager"), // 管理员 DEVELOPER("developer"), // 开发人员 MAINTAINER("maintainer"), // 运维人员 TESTER("tester"), // 测试人员 PM("pm"), // 产品人员 - QC("qc"); // 质量管理员 + QC("qc"), // 质量管理员 + CI_MANAGER("ci_manager"); // CI 管理员 companion object { fun get(value: String): BkAuthGroup { diff --git a/src/backend/ci/core/common/common-client/build.gradle.kts b/src/backend/ci/core/common/common-client/build.gradle.kts index 1cdc8993a5b..aecc38a34d5 100644 --- a/src/backend/ci/core/common/common-client/build.gradle.kts +++ b/src/backend/ci/core/common/common-client/build.gradle.kts @@ -33,4 +33,8 @@ dependencies { api("io.github.openfeign.form:feign-form") api("io.github.openfeign.form:feign-form-spring") api("io.github.openfeign:feign-spring4") + if (System.getProperty("devops.assemblyMode") == "KUBERNETES") { + print("use common-kubernetes") + api(project(":core:common:common-kubernetes")) + } } diff --git a/src/backend/ci/core/common/common-client/src/main/kotlin/com/tencent/devops/common/client/ms/MicroServiceTarget.kt b/src/backend/ci/core/common/common-client/src/main/kotlin/com/tencent/devops/common/client/ms/MicroServiceTarget.kt index 73b15105837..330a51a9f25 100644 --- a/src/backend/ci/core/common/common-client/src/main/kotlin/com/tencent/devops/common/client/ms/MicroServiceTarget.kt +++ b/src/backend/ci/core/common/common-client/src/main/kotlin/com/tencent/devops/common/client/ms/MicroServiceTarget.kt @@ -37,7 +37,6 @@ import com.tencent.devops.common.service.utils.MessageCodeUtil import feign.Request import feign.RequestTemplate import org.apache.commons.lang3.RandomUtils -import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory import org.springframework.cloud.client.ServiceInstance import org.springframework.cloud.client.discovery.composite.CompositeDiscoveryClient @@ -69,14 +68,7 @@ class MicroServiceTarget constructor( val instances = if (KubernetesUtils.inContainer()) { val namespace = discoveryTag.replace("kubernetes-", "") - val pods = msCache.get(KubernetesUtils.getSvrName(serviceName)) - pods.filter { inNamespace(it.metadata, namespace) }.ifEmpty { - if (StringUtils.isNotBlank(KubernetesUtils.getDefaultNamespace())) { - pods.filter { inNamespace(it.metadata, KubernetesUtils.getDefaultNamespace()) } - } else { - emptyList() - } - } + msCache.get(KubernetesUtils.getSvrName(serviceName, namespace)) } else { msCache.get(serviceName).filter { it is ConsulServiceInstance && it.tags.contains(discoveryTag) } } @@ -87,20 +79,6 @@ class MicroServiceTarget constructor( return instances[RandomUtils.nextInt(0, instances.size)] } - /** - * 判断是否在集群中 - */ - private fun inNamespace(metadata: Map, namespace: String): Boolean { - for (entry in metadata) { - if (entry.key.contains("namespace")) { - if (entry.value == namespace) { - return true - } - } - } - return false - } - override fun apply(input: RequestTemplate?): Request { if (input!!.url().indexOf("http") != 0) { input.target(url()) diff --git a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/pojo/docker/DockerConstants.kt b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/pojo/docker/DockerConstants.kt index d4fbd496574..35dedc728d4 100644 --- a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/pojo/docker/DockerConstants.kt +++ b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/pojo/docker/DockerConstants.kt @@ -32,4 +32,12 @@ object DockerConstants { * docker路由Key */ const val DOCKER_ROUTING_KEY_PREFIX = "dispatchdocker:docker_routing" + + const val ENV_KEY_BUILD_ID = "devops_build_id" + const val ENV_KEY_PROJECT_ID = "devops_project_id" + const val ENV_KEY_AGENT_ID = "devops_agent_id" + const val ENV_KEY_AGENT_SECRET_KEY = "devops_agent_secret_key" + const val ENV_KEY_GATEWAY = "devops_gateway" + + const val ENV_JOB_BUILD_TYPE = "JOB_POOL" } diff --git a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt index 6791bfac2a2..fcfe349ea46 100644 --- a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt +++ b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/service/DispatchService.kt @@ -39,6 +39,10 @@ import com.tencent.devops.common.dispatch.sdk.DispatchSdkErrorCode import com.tencent.devops.common.dispatch.sdk.pojo.DispatchMessage import com.tencent.devops.common.dispatch.sdk.pojo.RedisBuild import com.tencent.devops.common.dispatch.sdk.pojo.SecretInfo +import com.tencent.devops.common.dispatch.sdk.pojo.docker.DockerConstants.ENV_KEY_AGENT_ID +import com.tencent.devops.common.dispatch.sdk.pojo.docker.DockerConstants.ENV_KEY_AGENT_SECRET_KEY +import com.tencent.devops.common.dispatch.sdk.pojo.docker.DockerConstants.ENV_KEY_BUILD_ID +import com.tencent.devops.common.dispatch.sdk.pojo.docker.DockerConstants.ENV_KEY_PROJECT_ID import com.tencent.devops.common.dispatch.sdk.utils.ChannelUtils import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher import com.tencent.devops.common.log.utils.BuildLogPrinter @@ -88,6 +92,13 @@ class DispatchService constructor( fun buildDispatchMessage(event: PipelineAgentStartupEvent): DispatchMessage { logger.info("[${event.buildId}] Start build with gateway - ($gateway)") val secretInfo = setRedisAuth(event) + + val customBuildEnv = event.customBuildEnv?.toMutableMap() ?: mutableMapOf() + customBuildEnv[ENV_KEY_BUILD_ID] = event.buildId + customBuildEnv[ENV_KEY_PROJECT_ID] = event.projectId + customBuildEnv[ENV_KEY_AGENT_ID] = secretInfo.hashId + customBuildEnv[ENV_KEY_AGENT_SECRET_KEY] = secretInfo.secretKey + return DispatchMessage( id = secretInfo.hashId, secretKey = secretInfo.secretKey, @@ -108,7 +119,7 @@ class DispatchService constructor( containerType = event.containerType, stageId = event.stageId, dispatchType = event.dispatchType, - customBuildEnv = event.customBuildEnv, + customBuildEnv = customBuildEnv, dockerRoutingType = event.dockerRoutingType ) } diff --git a/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentGrayUtils.kt b/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentGrayUtils.kt index 407d11afcc1..f4fb424889c 100644 --- a/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentGrayUtils.kt +++ b/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentGrayUtils.kt @@ -52,6 +52,9 @@ class AgentGrayUtils constructor( private const val CURRENT_AGENT_LINUX_MIPS64_JDK_VERSION = "environment.thirdparty.agent.linux_mips64_jdk.verison" + private const val CURRENT_AGENT_LINUX_AMD64_DOCKER_INIT_FILE_MD5 = + "environment.thirdparty.agent.linux_amd64.docker_init_file.md5" + private const val CAN_UPGRADE_AGENT_SET_KEY = "environment:thirdparty:can_upgrade" private const val LOCK_UPGRADE_AGENT_SET_KEY = "environment:thirdparty:lock_upgrade" @@ -59,12 +62,16 @@ class AgentGrayUtils constructor( private const val LOCK_UPGRADE_AGENT_WORKER_SET_KEY = "environment:thirdparty:worker:lock_upgrade" private const val LOCK_UPGRADE_AGENT_GO_SET_KEY = "environment:thirdparty:goagent:lock_upgrade" private const val LOCK_UPGRADE_AGENT_JDK_SET_KEY = "environment:thirdparty:jdk:lock_upgrade" + private const val LOCK_UPGRADE_DOCKER_INIT_FILE_SET_KEY = + "environment:thirdparty:docker_init_file:lock_upgrade" private const val FORCE_UPGRADE_AGENT_SET_KEY = "environment:thirdparty:force_upgrade" private const val FORCE_UPGRADE_AGENT_WORKER_SET_KEY = "environment:thirdparty:worker:force_upgrade" private const val FORCE_UPGRADE_AGENT_GO_SET_KEY = "environment:thirdparty:goagent:force_upgrade" private const val FORCE_UPGRADE_AGENT_JDK_SET_KEY = "environment:thirdparty:jdk:force_upgrade" + private const val FORCE_UPGRADE_AGENT_DOCKER_INIT_FILE_SET_KEY = + "environment:thirdparty:docker_init_file:force_upgrade" private const val DEFAULT_GATEWAY_KEY = "environment:thirdparty:default_gateway" private const val DEFAULT_FILE_GATEWAY_KEY = "environment:thirdparty:default_file_gateway" @@ -114,6 +121,7 @@ class AgentGrayUtils constructor( AgentUpgradeType.WORKER -> FORCE_UPGRADE_AGENT_WORKER_SET_KEY AgentUpgradeType.GO_AGENT -> FORCE_UPGRADE_AGENT_GO_SET_KEY AgentUpgradeType.JDK -> FORCE_UPGRADE_AGENT_JDK_SET_KEY + AgentUpgradeType.DOCKER_INIT_FILE -> FORCE_UPGRADE_AGENT_DOCKER_INIT_FILE_SET_KEY } } @@ -155,6 +163,7 @@ class AgentGrayUtils constructor( AgentUpgradeType.WORKER -> LOCK_UPGRADE_AGENT_WORKER_SET_KEY AgentUpgradeType.GO_AGENT -> LOCK_UPGRADE_AGENT_GO_SET_KEY AgentUpgradeType.JDK -> LOCK_UPGRADE_AGENT_JDK_SET_KEY + AgentUpgradeType.DOCKER_INIT_FILE -> LOCK_UPGRADE_DOCKER_INIT_FILE_SET_KEY } } @@ -226,6 +235,10 @@ class AgentGrayUtils constructor( } } + fun getDockerInitFileMd5Key(): String { + return CURRENT_AGENT_LINUX_AMD64_DOCKER_INIT_FILE_MD5 + } + fun getDefaultGateway(): String? { return redisOperation.get(DEFAULT_GATEWAY_KEY) } diff --git a/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentUpgradeType.kt b/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentUpgradeType.kt index 8930399e3c0..010a79fa114 100644 --- a/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentUpgradeType.kt +++ b/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentUpgradeType.kt @@ -1,7 +1,7 @@ package com.tencent.devops.common.environment.agent enum class AgentUpgradeType { - WORKER, GO_AGENT, JDK; + WORKER, GO_AGENT, JDK, DOCKER_INIT_FILE; companion object { fun find(type: String?): AgentUpgradeType? { @@ -9,6 +9,7 @@ enum class AgentUpgradeType { WORKER.name -> WORKER GO_AGENT.name -> GO_AGENT JDK.name -> JDK + DOCKER_INIT_FILE.name -> DOCKER_INIT_FILE else -> null } } diff --git a/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/ThirdPartyAgentHeartbeatUtils.kt b/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/ThirdPartyAgentHeartbeatUtils.kt index bf079a9db82..2ff68a7f575 100644 --- a/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/ThirdPartyAgentHeartbeatUtils.kt +++ b/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/ThirdPartyAgentHeartbeatUtils.kt @@ -46,12 +46,19 @@ class ThirdPartyAgentHeartbeatUtils constructor( fun saveNewHeartbeat(projectId: String, agentId: Long, newHeartbeatInfo: NewHeartbeatInfo) { // #5806 设置数量 - newHeartbeatInfo.busyTaskSize = newHeartbeatInfo.taskList.size + newHeartbeatInfo.busyTaskSize = newHeartbeatInfo.taskList?.size ?: 0 + + newHeartbeatInfo.dockerBusyTaskSize = newHeartbeatInfo.dockerTaskList?.size ?: 0 // #5806 防止被塞爆，数量过大就不支持展示 - if (newHeartbeatInfo.taskList.size > MAX_TASKS) { - newHeartbeatInfo.taskList = newHeartbeatInfo.taskList.subList(0, MAX_TASKS) + if (newHeartbeatInfo.busyTaskSize > MAX_TASKS) { + newHeartbeatInfo.taskList = newHeartbeatInfo.taskList?.subList(0, MAX_TASKS) + } + + if (newHeartbeatInfo.dockerBusyTaskSize > MAX_TASKS) { + newHeartbeatInfo.dockerTaskList = newHeartbeatInfo.dockerTaskList?.subList(0, MAX_TASKS) } + newHeartbeatInfo.projectId = projectId newHeartbeatInfo.agentId = agentId newHeartbeatInfo.heartbeatTime = System.currentTimeMillis() diff --git a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQEventDispatcher.kt b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQEventDispatcher.kt index 28d65e7f78c..b3c498b4215 100644 --- a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQEventDispatcher.kt +++ b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQEventDispatcher.kt @@ -27,6 +27,8 @@ package com.tencent.devops.common.event.dispatcher.pipeline.mq +import com.rabbitmq.client.ChannelContinuationTimeoutException +import com.rabbitmq.client.impl.AMQImpl import com.tencent.devops.common.event.annotation.Event import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher import com.tencent.devops.common.event.pojo.pipeline.IPipelineEvent @@ -43,35 +45,46 @@ class MQEventDispatcher constructor( private val rabbitTemplate: RabbitTemplate ) : PipelineEventDispatcher { + @SuppressWarnings("NestedBlockDepth") override fun dispatch(vararg events: IPipelineEvent) { events.forEach { event -> try { - val eventType = event::class.java.annotations.find { s -> s is Event } as Event - val routeKey = // 根据 routeKey+后缀 实现动态变换路由Key - if (event is IPipelineRoutableEvent && !event.routeKeySuffix.isNullOrBlank()) { - eventType.routeKey + event.routeKeySuffix - } else { - eventType.routeKey - } -// logger.info("dispatch the event|Route=$routeKey|exchange=${eventType.exchange}" + -// "|source=(${event.javaClass.name}:${event.source}-${event.actionType}-${event.pipelineId})") - rabbitTemplate.convertAndSend(eventType.exchange, routeKey, event) { message -> - // 事件中的变量指定 - when { - event.delayMills > 0 -> message.messageProperties.setHeader("x-delay", event.delayMills) - eventType.delayMills > 0 -> // 事件类型固化默认值 - message.messageProperties.setHeader("x-delay", eventType.delayMills) - else -> // 非延时消息的则8小时后过期，防止意外发送的消息无消费端ACK处理从而堆积过多消息导致MQ故障 - message.messageProperties.expiration = "28800000" + send(event) + } catch (ignored: Exception) { + if (ignored.cause is ChannelContinuationTimeoutException) { + logger.warn("[ENGINE_MQ_SEVERE]Fail to dispatch the event($event)", ignored) + val cause = ignored.cause as ChannelContinuationTimeoutException + if (cause.method is AMQImpl.Channel.Open) { + send(event) } - message + } else { + logger.error("[ENGINE_MQ_SEVERE]Fail to dispatch the event($event)", ignored) } - } catch (ignored: Exception) { - logger.error("[ENGINE_MQ_SEVERE]Fail to dispatch the event($event)", ignored) } } } + private fun send(event: IPipelineEvent) { + val eventType = event::class.java.annotations.find { s -> s is Event } as Event + val routeKey = // 根据 routeKey+后缀 实现动态变换路由Key + if (event is IPipelineRoutableEvent && !event.routeKeySuffix.isNullOrBlank()) { + eventType.routeKey + event.routeKeySuffix + } else { + eventType.routeKey + } + rabbitTemplate.convertAndSend(eventType.exchange, routeKey, event) { message -> + // 事件中的变量指定 + when { + event.delayMills > 0 -> message.messageProperties.setHeader("x-delay", event.delayMills) + eventType.delayMills > 0 -> // 事件类型固化默认值 + message.messageProperties.setHeader("x-delay", eventType.delayMills) + else -> // 非延时消息的则8小时后过期，防止意外发送的消息无消费端ACK处理从而堆积过多消息导致MQ故障 + message.messageProperties.expiration = "28800000" + } + message + } + } + companion object { private val logger = LoggerFactory.getLogger(MQEventDispatcher::class.java) } diff --git a/src/backend/ci/core/common/common-kubernetes/build.gradle.kts b/src/backend/ci/core/common/common-kubernetes/build.gradle.kts new file mode 100644 index 00000000000..644c8193ac4 --- /dev/null +++ b/src/backend/ci/core/common/common-kubernetes/build.gradle.kts @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +dependencies { + api("org.springframework.cloud:spring-cloud-starter-kubernetes-client") +} diff --git a/src/backend/ci/core/common/common-kubernetes/src/main/kotlin/com/tencent/devops/common/kubernetes/client/BkKubernetesDiscoveryClient.kt b/src/backend/ci/core/common/common-kubernetes/src/main/kotlin/com/tencent/devops/common/kubernetes/client/BkKubernetesDiscoveryClient.kt new file mode 100644 index 00000000000..60432ec2a5f --- /dev/null +++ b/src/backend/ci/core/common/common-kubernetes/src/main/kotlin/com/tencent/devops/common/kubernetes/client/BkKubernetesDiscoveryClient.kt @@ -0,0 +1,78 @@ +package com.tencent.devops.common.kubernetes.client + +import io.kubernetes.client.informer.SharedInformer +import io.kubernetes.client.informer.SharedInformerFactory +import io.kubernetes.client.informer.cache.Lister +import io.kubernetes.client.openapi.models.V1Endpoints +import io.kubernetes.client.openapi.models.V1Service +import org.slf4j.LoggerFactory +import org.springframework.cloud.client.ServiceInstance +import org.springframework.cloud.kubernetes.client.discovery.KubernetesInformerDiscoveryClient +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesServiceInstance +import org.springframework.util.StringUtils + +@SuppressWarnings("LongParameterList", "ReturnCount") +class BkKubernetesDiscoveryClient constructor( + namespace: String, + sharedInformerFactory: SharedInformerFactory, + private val serviceLister: Lister, + private val endpointsLister: Lister, + serviceInformer: SharedInformer, + endpointsInformer: SharedInformer, + properties: KubernetesDiscoveryProperties +) : KubernetesInformerDiscoveryClient( + namespace, + sharedInformerFactory, + serviceLister, + endpointsLister, + serviceInformer, + endpointsInformer, + properties +) { + override fun getInstances(serviceId: String?): List { + if (!StringUtils.hasText(serviceId)) { + logger.error("ServiceId is null or empty , please check the serviceId") + return emptyList() + } + if (!serviceId!!.contains(".")) { + logger.error("ServiceId must contain '.' , the kubernetes svc must {serviceName}.{namespace}") + return emptyList() + } + val serviceData = serviceId.split('.') + val serviceName = serviceData[0] + val namespace = serviceData[1] + val service = serviceLister.namespace(namespace).get(serviceName) + if (service == null) { + logger.debug("Can not find service , service id : $serviceId") + return emptyList() + } + val svcMetadata = mutableMapOf() + service.metadata?.labels?.let { svcMetadata.putAll(it) } + service.metadata?.annotations?.let { svcMetadata.putAll(it) } + + val ep = endpointsLister.namespace(service.metadata!!.namespace)[service.metadata!!.name] + if (ep == null || ep.subsets == null) { + logger.debug("Can not find endpoint , service id : $serviceId") + return emptyList() + } + return ep.subsets!!.filterNot { it.ports.isNullOrEmpty() }.flatMap { + val endpointPort = it.ports?.get(0)?.port ?: 80 + val addresses = it.addresses ?: emptyList() + addresses.map { addr -> + KubernetesServiceInstance( + addr.targetRef?.uid ?: "", + serviceName, + addr.ip, + endpointPort, + svcMetadata, + false + ) + } + }.toList() + } + + companion object { + private val logger = LoggerFactory.getLogger(BkKubernetesDiscoveryClient::class.java) + } +} diff --git a/src/backend/ci/core/common/common-kubernetes/src/main/kotlin/com/tencent/devops/common/kubernetes/config/BkKubernetesConfiguration.kt b/src/backend/ci/core/common/common-kubernetes/src/main/kotlin/com/tencent/devops/common/kubernetes/config/BkKubernetesConfiguration.kt new file mode 100644 index 00000000000..a692879104f --- /dev/null +++ b/src/backend/ci/core/common/common-kubernetes/src/main/kotlin/com/tencent/devops/common/kubernetes/config/BkKubernetesConfiguration.kt @@ -0,0 +1,52 @@ +package com.tencent.devops.common.kubernetes.config + +import com.tencent.devops.common.kubernetes.client.BkKubernetesDiscoveryClient +import io.kubernetes.client.informer.SharedInformer +import io.kubernetes.client.informer.SharedInformerFactory +import io.kubernetes.client.informer.cache.Lister +import io.kubernetes.client.openapi.models.V1Endpoints +import io.kubernetes.client.openapi.models.V1Service +import org.slf4j.LoggerFactory +import org.springframework.boot.autoconfigure.AutoConfigureAfter +import org.springframework.boot.autoconfigure.AutoConfigureBefore +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.cloud.kubernetes.client.KubernetesClientAutoConfiguration +import org.springframework.cloud.kubernetes.client.discovery.ConditionalOnKubernetesDiscoveryEnabled +import org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration +import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Primary + +@Configuration +@ConditionalOnKubernetesDiscoveryEnabled +@AutoConfigureBefore(KubernetesDiscoveryClientAutoConfiguration::class) +@AutoConfigureAfter(KubernetesClientAutoConfiguration::class) +@EnableConfigurationProperties(KubernetesDiscoveryProperties::class) +class BkKubernetesConfiguration { + @Bean + @Primary + @SuppressWarnings("LongParameterList") + fun kubernetesInformerDiscoveryClient( + kubernetesNamespaceProvider: KubernetesNamespaceProvider, + sharedInformerFactory: SharedInformerFactory, + serviceLister: Lister, + endpointsLister: Lister, + serviceInformer: SharedInformer, + endpointsInformer: SharedInformer, + properties: KubernetesDiscoveryProperties + ): BkKubernetesDiscoveryClient { + logger.debug("properties allNamespaces : ${properties.isAllNamespaces}") + logger.info("kubernetesInformerDiscoveryClient init success") + return BkKubernetesDiscoveryClient( + kubernetesNamespaceProvider.namespace, + sharedInformerFactory, serviceLister, endpointsLister, serviceInformer, endpointsInformer, + properties + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(BkKubernetesConfiguration::class.java) + } +} diff --git a/src/backend/ci/core/common/common-kubernetes/src/main/resources/META-INF/spring.factories b/src/backend/ci/core/common/common-kubernetes/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000000..a37ded5008b --- /dev/null +++ b/src/backend/ci/core/common/common-kubernetes/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.devops.common.kubernetes.config.BkKubernetesConfiguration diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/Model.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/Model.kt index 8b07e9f5719..e0af3495d03 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/Model.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/Model.kt @@ -59,7 +59,9 @@ data class Model( @ApiModelProperty("提示", required = false) var tips: String? = null, @ApiModelProperty("流水线事件回调", required = false) - var events: Map? = emptyMap() + var events: Map? = emptyMap(), + @ApiModelProperty("静态流水线组", required = false) + var staticViews: List = emptyList() ) { @ApiModelProperty("提交时流水线最新版本号", required = false) var latestVersion: Int = 0 @@ -108,6 +110,7 @@ data class Model( jobId = container.jobId ) } + is NormalContainer -> { NormalContainer( containerId = container.containerId, @@ -127,6 +130,7 @@ data class Model( jobId = container.jobId ) } + else -> { container } diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/enums/BuildStatus.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/enums/BuildStatus.kt index 238a7a26410..ee83af08847 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/enums/BuildStatus.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/enums/BuildStatus.kt @@ -73,6 +73,10 @@ enum class BuildStatus(val statusName: String, val visible: Boolean) { fun isCancel(): Boolean = this == CANCELED + fun isSkip(): Boolean = this == SKIP + + fun isTerminate(): Boolean = this == TERMINATE + fun isRunning(): Boolean = this == RUNNING || this == LOOP_WAITING || diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/Element.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/Element.kt index 80f5a86207f..6d78cc96314 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/Element.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/Element.kt @@ -56,6 +56,7 @@ import com.tencent.devops.common.pipeline.pojo.element.trigger.ManualTriggerElem import com.tencent.devops.common.pipeline.pojo.element.trigger.RemoteTriggerElement import com.tencent.devops.common.pipeline.pojo.element.trigger.TimerTriggerElement import com.tencent.devops.common.pipeline.utils.SkipElementUtils +import io.swagger.annotations.ApiModel import io.swagger.annotations.ApiModelProperty @JsonTypeInfo( @@ -94,6 +95,7 @@ import io.swagger.annotations.ApiModelProperty JsonSubTypes.Type(value = CodeP4WebHookTriggerElement::class, name = CodeP4WebHookTriggerElement.classType) ) @Suppress("ALL") +@ApiModel("Element 基类") abstract class Element( @ApiModelProperty("任务名称", required = false) open val name: String, diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/CodeTGitWebHookTriggerElement.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/CodeTGitWebHookTriggerElement.kt index e083ce9bcc3..4cb174d9369 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/CodeTGitWebHookTriggerElement.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/CodeTGitWebHookTriggerElement.kt @@ -65,6 +65,7 @@ data class CodeTGitWebHookTriggerData( val input: CodeTGitWebHookTriggerInput ) +@ApiModel("TGit事件触发数据") data class CodeTGitWebHookTriggerInput( @ApiModelProperty("仓库ID", required = true) val repositoryHashId: String?, diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/BuildType.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/BuildType.kt index ba561efda41..a4cd064def3 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/BuildType.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/BuildType.kt @@ -28,7 +28,6 @@ package com.tencent.devops.common.pipeline.type import com.tencent.devops.common.api.pojo.OS -import com.tencent.devops.common.service.utils.KubernetesUtils enum class BuildType( val value: String, @@ -40,29 +39,15 @@ enum class BuildType( ESXi("蓝盾公共构建资源", listOf(OS.MACOS), false, false, false), MACOS("蓝盾公共构建资源(NEW)", listOf(OS.MACOS), false, false, false), WINDOWS("云托管：Windows on DevCloud", listOf(OS.WINDOWS), false, false, false), - KUBERNETES( - "Kubernetes构建资源", - listOf(OS.LINUX), - KubernetesUtils.enableK8sBuild(), - KubernetesUtils.enableK8sBuild(), - KubernetesUtils.enableK8sBuild() - ), - IDC("公共：Docker on IDC CVM", listOf(OS.LINUX), true, false, false), + KUBERNETES("Kubernetes构建资源", listOf(OS.LINUX), false, false, false), PUBLIC_DEVCLOUD("公共：Docker on DevCloud", listOf(OS.LINUX), true, false, false), PUBLIC_BCS("公共：Docker on Bcs", listOf(OS.LINUX), false, false, false), - TSTACK("Windows构建", listOf(OS.WINDOWS), false, false, false), // tstack is deleted THIRD_PARTY_AGENT_ID("私有：单构建机", listOf(OS.MACOS, OS.LINUX, OS.WINDOWS), false, true, true), THIRD_PARTY_AGENT_ENV("私有：构建集群", listOf(OS.MACOS, OS.LINUX, OS.WINDOWS), false, true, true), THIRD_PARTY_PCG("PCG公共构建资源", listOf(OS.LINUX), false, false, false), THIRD_PARTY_DEVCLOUD("腾讯自研云（云devnet资源）", listOf(OS.LINUX), false, false, false), GIT_CI("工蜂CI", listOf(OS.LINUX), false, false, false), - DOCKER( - "Docker公共构建机", - listOf(OS.LINUX), - KubernetesUtils.enablePublicDocker(), - KubernetesUtils.enablePublicDocker(), - KubernetesUtils.enablePublicDocker() - ), + DOCKER("Docker公共构建机", listOf(OS.LINUX), true, true, true), STREAM("stream", listOf(OS.LINUX), false, false, false), AGENT_LESS("无编译环境", listOf(OS.LINUX), false, false, false) } diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/DispatchType.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/DispatchType.kt index 5f7aebf4d17..d5f45b6a06f 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/DispatchType.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/DispatchType.kt @@ -39,14 +39,12 @@ import com.tencent.devops.common.pipeline.type.codecc.CodeCCDispatchType import com.tencent.devops.common.pipeline.type.docker.DockerDispatchType import com.tencent.devops.common.pipeline.type.exsi.ESXiDispatchType import com.tencent.devops.common.pipeline.type.kubernetes.KubernetesDispatchType -import com.tencent.devops.common.pipeline.type.tstack.TStackDispatchType @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "buildType", visible = false) @JsonSubTypes( JsonSubTypes.Type(value = DockerDispatchType::class, name = "DOCKER"), JsonSubTypes.Type(value = KubernetesDispatchType::class, name = "KUBERNETES"), JsonSubTypes.Type(value = ESXiDispatchType::class, name = "ESXi"), - JsonSubTypes.Type(value = TStackDispatchType::class, name = "TSTACK"), JsonSubTypes.Type(value = ThirdPartyAgentIDDispatchType::class, name = "THIRD_PARTY_AGENT_ID"), JsonSubTypes.Type(value = ThirdPartyAgentEnvDispatchType::class, name = "THIRD_PARTY_AGENT_ENV"), JsonSubTypes.Type(value = ThirdPartyDevCloudDispatchType::class, name = "THIRD_PARTY_DEVCLOUD"), diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentDockerInfo.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentDockerInfo.kt new file mode 100644 index 00000000000..c8143b7d982 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentDockerInfo.kt @@ -0,0 +1,33 @@ +package com.tencent.devops.common.pipeline.type.agent + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonInclude + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +data class ThirdPartyAgentDockerInfo( + val image: String, + val credential: Credential?, + val envs: Map? +) + +data class Credential( + val user: String, + val password: String +) + +// 第三方构建机docker类型，调度使用，会带有调度相关信息 +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +data class ThirdPartyAgentDockerInfoDispatch( + // docker类型构建机需要，调度 + val agentId: String, + val secretKey: String, + val image: String, + val credential: Credential?, + val envs: Map? +) { + constructor(agentId: String, secretKey: String, info: ThirdPartyAgentDockerInfo) : this( + agentId, secretKey, info.image, info.credential, info.envs + ) +} diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentEnvDispatchType.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentEnvDispatchType.kt index 597510ea5ae..e9483070175 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentEnvDispatchType.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentEnvDispatchType.kt @@ -31,12 +31,19 @@ import com.fasterxml.jackson.annotation.JsonProperty import com.tencent.devops.common.api.util.EnvUtils import com.tencent.devops.common.pipeline.type.BuildType import com.tencent.devops.common.pipeline.type.DispatchType +import io.swagger.annotations.ApiModelProperty data class ThirdPartyAgentEnvDispatchType( - @JsonProperty("value") var envName: String, + @JsonProperty("value") + var envName: String, + @ApiModelProperty("共享环境时必填，值为提供共享环境的项目id") var envProjectId: String?, + @ApiModelProperty("工作空间") var workspace: String?, - val agentType: AgentType = AgentType.NAME + @ApiModelProperty("agent类型,默认NAME") + val agentType: AgentType = AgentType.NAME, + // 第三方构建机用docker作为构建机 + val dockerInfo: ThirdPartyAgentDockerInfo? ) : DispatchType( envName ) { diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentIDDispatchType.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentIDDispatchType.kt index 0164b7b7846..5aca0163777 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentIDDispatchType.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentIDDispatchType.kt @@ -35,7 +35,9 @@ import com.tencent.devops.common.pipeline.type.DispatchType data class ThirdPartyAgentIDDispatchType( @JsonProperty("value") var displayName: String, var workspace: String?, - val agentType: AgentType = AgentType.NAME + val agentType: AgentType = AgentType.NAME, + // 第三方构建机用docker作为构建机 + val dockerInfo: ThirdPartyAgentDockerInfo? ) : DispatchType( displayName ) { diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/bcs/PublicBcsDispatchType.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/bcs/PublicBcsDispatchType.kt index ebf90025fa9..fd183b36210 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/bcs/PublicBcsDispatchType.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/bcs/PublicBcsDispatchType.kt @@ -6,6 +6,7 @@ import com.tencent.devops.common.pipeline.type.BuildType import com.tencent.devops.common.pipeline.type.DispatchRouteKeySuffix import com.tencent.devops.common.pipeline.type.StoreDispatchType import com.tencent.devops.common.pipeline.type.docker.ImageType +import io.swagger.annotations.ApiModelProperty data class PublicBcsDispatchType( @JsonProperty("value") var image: String?, @@ -13,11 +14,11 @@ data class PublicBcsDispatchType( override var imageType: ImageType? = ImageType.BKDEVOPS, override var credentialId: String? = "", override var credentialProject: String? = "", - // 商店镜像代码 + @ApiModelProperty("商店镜像代码") override var imageCode: String? = "", - // 商店镜像版本 + @ApiModelProperty("商店镜像版本") override var imageVersion: String? = "", - // 商店镜像名称 + @ApiModelProperty("商店镜像名称") override var imageName: String? = "" ) : StoreDispatchType( dockerBuildVersion = if (image.isNullOrBlank()) diff --git a/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisLock.kt b/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisLock.kt index 4cdf8d54ac4..a34af7c0ece 100644 --- a/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisLock.kt +++ b/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisLock.kt @@ -189,6 +189,15 @@ open class RedisLock( return true } + fun lockAround(action: () -> T): T { + try { + this.lock() + return action() + } finally { + this.unlock() + } + } + override fun close() { unlock() } diff --git a/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisOperation.kt b/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisOperation.kt index 1461ced1d8d..26a3a1e2862 100644 --- a/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisOperation.kt +++ b/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisOperation.kt @@ -81,6 +81,25 @@ class RedisOperation(private val redisTemplate: RedisTemplate, p } } + fun setIfAbsent( + key: String, + value: String, + expiredInSecond: Long? = null, + expired: Boolean? = true, + isDistinguishCluster: Boolean? = false + ): Boolean { + val finalKey = getFinalKey(key, isDistinguishCluster) + return if (expired == false) { + redisTemplate.opsForValue().setIfAbsent(finalKey, value) ?: false + } else { + var timeout = expiredInSecond ?: maxExpireTime + if (timeout <= 0) { // #5901 不合法值清理，设置默认为超时时间，防止出错。 + timeout = maxExpireTime + } + redisTemplate.opsForValue().setIfAbsent(finalKey, value, timeout, TimeUnit.SECONDS) ?: false + } + } + fun delete(key: String, isDistinguishCluster: Boolean? = false) { redisTemplate.delete(getFinalKey(key, isDistinguishCluster)) } @@ -244,6 +263,10 @@ class RedisOperation(private val redisTemplate: RedisTemplate, p return redisTemplate.opsForList().rightPop(getFinalKey(key, isDistinguishCluster)) } + fun trim(key: String, start: Long, end: Long) { + redisTemplate.opsForList().trim(key, start, end) + } + fun getRedisName(): String? { return redisName } diff --git a/src/backend/ci/core/common/common-scm/src/main/kotlin/com/tencent/devops/scm/code/CodeGitScmOauthImpl.kt b/src/backend/ci/core/common/common-scm/src/main/kotlin/com/tencent/devops/scm/code/CodeGitScmOauthImpl.kt index 91fb04212a1..d33dff1a260 100644 --- a/src/backend/ci/core/common/common-scm/src/main/kotlin/com/tencent/devops/scm/code/CodeGitScmOauthImpl.kt +++ b/src/backend/ci/core/common/common-scm/src/main/kotlin/com/tencent/devops/scm/code/CodeGitScmOauthImpl.kt @@ -57,7 +57,7 @@ class CodeGitScmOauthImpl constructor( private val event: String? = null ) : IScm { - private val apiUrl = GitUtils.getGitApiUrl(apiUrl = gitConfig.gitApiUrl, repoUrl = url) + private val apiUrl = gitConfig.gitApiUrl override fun getLatestRevision(): RevisionInfo { val branch = branchName ?: "master" diff --git a/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/KubernetesUtils.kt b/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/KubernetesUtils.kt index a81f55c8698..c9a91cc5e28 100644 --- a/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/KubernetesUtils.kt +++ b/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/KubernetesUtils.kt @@ -9,19 +9,6 @@ object KubernetesUtils { private val chartName = System.getenv("CHART_NAME") private val multiCluster = BooleanUtils.toBoolean(System.getenv("MULTI_CLUSTER")) private val defaultNamespace = System.getenv("DEFAULT_NAMESPACE") - private val enablePublicDocker = - BooleanUtils.toBoolean(StringUtils.defaultIfBlank(System.getenv("ENABLE_PUBLIC_DOCKER"), "true")) - private val enableK8sBuild = - BooleanUtils.toBoolean(StringUtils.defaultIfBlank(System.getenv("ENABLE_K8S_BUILD"), "false")) - - /** - * 是否开启docker公共构建机 - */ - fun enablePublicDocker() = enablePublicDocker - /** - * 是否开启k8s构建机 - */ - fun enableK8sBuild() = enableK8sBuild /** * 服务是否在容器中 @@ -36,11 +23,11 @@ object KubernetesUtils { /** * 获取服务发现的名称 */ - fun getSvrName(serviceName: String): String { + fun getSvrName(serviceName: String, namespace: String): String { return if (multiCluster) { - "$serviceName-$chartName-$serviceName" + "$serviceName-$chartName-$serviceName.$namespace" } else { - "$releaseName-$chartName-$serviceName" + "$releaseName-$chartName-$serviceName.$namespace" } } diff --git a/src/backend/ci/core/common/common-test/build.gradle.kts b/src/backend/ci/core/common/common-test/build.gradle.kts index 06fba7688c3..1385cb699b4 100644 --- a/src/backend/ci/core/common/common-test/build.gradle.kts +++ b/src/backend/ci/core/common/common-test/build.gradle.kts @@ -28,4 +28,8 @@ dependencies { api("org.mockito:mockito-all") api("com.nhaarman:mockito-kotlin-kt1.1") + api("io.mockk:mockk") + api(project(":core:common:common-service")) + api(project(":core:common:common-client")) + api("org.junit.jupiter:junit-jupiter-api") } diff --git a/src/backend/ci/core/common/common-test/src/main/kotlin/com/tencent/devops/common/test/BkCiAbstractTest.kt b/src/backend/ci/core/common/common-test/src/main/kotlin/com/tencent/devops/common/test/BkCiAbstractTest.kt new file mode 100644 index 00000000000..acfeacd1512 --- /dev/null +++ b/src/backend/ci/core/common/common-test/src/main/kotlin/com/tencent/devops/common/test/BkCiAbstractTest.kt @@ -0,0 +1,110 @@ +package com.tencent.devops.common.test + +import com.fasterxml.jackson.databind.ObjectMapper +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.redis.RedisOperation +import io.mockk.MockKMatcherScope +import io.mockk.every +import io.mockk.mockk +import io.mockk.spyk +import org.apache.commons.lang3.reflect.MethodUtils +import org.jooq.DSLContext +import org.jooq.Record +import org.jooq.Result +import org.jooq.SQLDialect +import org.jooq.Table +import org.jooq.impl.DSL +import org.jooq.tools.jdbc.Mock +import org.jooq.tools.jdbc.MockConnection +import org.junit.jupiter.api.BeforeAll +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.data.redis.core.RedisCallback +import java.lang.reflect.InvocationTargetException +import kotlin.reflect.KClass + +open class BkCiAbstractTest { + val dslContext: DSLContext = DSL.using(MockConnection(Mock.of(0)), SQLDialect.MYSQL) + val objectMapper: ObjectMapper = spyk() + + /** + * Mock JooQ 的返回 + * records 不传值 , Result为空 + */ + fun DSLContext.mockResult(t: Table, vararg records: R): Result { + val result = newResult(t) + records.forEach { result.add(it) } + return result + } + + /** + * dslContext的any() + */ + fun MockKMatcherScope.anyDslContext(): DSLContext = any() as DSLContext + + /** + * 调用私有函数 + */ + inline fun Any.invokePrivate(methodName: String, vararg args: Any): R? { + try { + val invokeResult = MethodUtils.invokeMethod(this, true, methodName, *args) ?: return null + if (invokeResult is R) { + return invokeResult + } else { + throw IllegalArgumentException("Result type is illegal") + } + } catch (e: Throwable) { + throw if (e is InvocationTargetException) e.targetException else e + } + } + + inline fun Client.mockGet(clz: KClass): T { + if (this !== client) { + logger.error("Just mock client can call mockGet") + throw RuntimeException() + } + var mockResourceMap = mockResourceMapThreadLocal.get() + if (null == mockResourceMap) { + mockResourceMap = mutableMapOf() + mockResourceMapThreadLocal.set(mockResourceMap) + } + if (mockResourceMap.contains(clz)) { + return mockResourceMap[clz] as T + } + val mockResource = mockk() + mockResourceMap[clz] = mockResource + return mockResource + } + + companion object { + val logger: Logger = LoggerFactory.getLogger("BKCI_JUNIT_TEST_LOGGER") + val redisOperation: RedisOperation = mockk(relaxed = true) + val client: Client = mockk(relaxed = true) + val mockResourceMapThreadLocal = ThreadLocal, Any>>() + + @JvmStatic + @BeforeAll + fun mockGet() { + every { client.get(any() as KClass<*>) } answers { + val clz = args[0] as KClass<*> + return@answers mockResourceMapThreadLocal.get()[clz]!! + } + } + + @JvmStatic + @BeforeAll + @SuppressWarnings("TooGenericExceptionThrown") + fun mockRedisOperation() { + every { redisOperation.execute(any>()) } answers { + val argStr = args[0]!!::class.toString() + if (argStr.contains("RedisLock\$set")) { + return@answers "OK" + } else if (argStr.contains("RedisLock\$unlock")) { + return@answers true + } else { + throw Exception("redisOperation.execute must mock by self") + } + } + } + } +} diff --git a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/handler/MissingKotlinParameterExceptionMapper.kt b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/handler/MissingKotlinParameterExceptionMapper.kt index 671c2209d96..e28bebfdac1 100644 --- a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/handler/MissingKotlinParameterExceptionMapper.kt +++ b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/handler/MissingKotlinParameterExceptionMapper.kt @@ -49,7 +49,7 @@ class MissingKotlinParameterExceptionMapper : ExceptionMapper { checkParam(projectId, agentId, secretKey) return thirdPartyAgentBuildService.checkIfCanUpgradeByVersionNew( - projectId, - agentId, - secretKey, - info + projectId = projectId, + agentId = agentId, + secretKey = secretKey, + info = info ) } diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/dao/ThirdPartyAgentBuildDao.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/dao/ThirdPartyAgentBuildDao.kt index cd845612870..600e1e92591 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/dao/ThirdPartyAgentBuildDao.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/dao/ThirdPartyAgentBuildDao.kt @@ -27,6 +27,8 @@ package com.tencent.devops.dispatch.dao +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentDockerInfoDispatch import com.tencent.devops.dispatch.pojo.enums.PipelineTaskStatus import com.tencent.devops.model.dispatch.tables.TDispatchThirdpartyAgentBuild import com.tencent.devops.model.dispatch.tables.records.TDispatchThirdpartyAgentBuildRecord @@ -34,8 +36,10 @@ import org.jooq.DSLContext import org.jooq.Result import org.springframework.stereotype.Repository import java.time.LocalDateTime +import org.jooq.JSON -@Repository@Suppress("ALL") +@Repository +@Suppress("ALL") class ThirdPartyAgentBuildDao { fun get(dslContext: DSLContext, buildId: String, vmSeqId: String): TDispatchThirdpartyAgentBuildRecord? { @@ -67,7 +71,10 @@ class ThirdPartyAgentBuildDao { buildNum: Int, taskName: String, agentIp: String, - nodeId: Long + nodeId: Long, + dockerInfo: ThirdPartyAgentDockerInfoDispatch?, + executeCount: Int?, + containerHashId: String? ): Int { with(TDispatchThirdpartyAgentBuild.T_DISPATCH_THIRDPARTY_AGENT_BUILD) { val now = LocalDateTime.now() @@ -85,6 +92,15 @@ class ThirdPartyAgentBuildDao { .set(STATUS, PipelineTaskStatus.QUEUE.status) .set(AGENT_IP, agentIp) .set(NODE_ID, nodeId) + .set( + DOCKER_INFO, if (dockerInfo == null) { + null + } else { + JSON.json(JsonUtil.toJson(dockerInfo, formatted = false)) + } + ) + .set(EXECUTE_COUNT, executeCount) + .set(CONTAINER_HASH_ID, containerHashId) .where(ID.eq(preRecord.id)).execute() } return dslContext.insertInto( @@ -102,7 +118,10 @@ class ThirdPartyAgentBuildDao { BUILD_NUM, TASK_NAME, AGENT_IP, - NODE_ID + NODE_ID, + DOCKER_INFO, + EXECUTE_COUNT, + CONTAINER_HASH_ID ).values( projectId, agentId, @@ -117,7 +136,14 @@ class ThirdPartyAgentBuildDao { buildNum, taskName, agentIp, - nodeId + nodeId, + if (dockerInfo == null) { + null + } else { + JSON.json(JsonUtil.toJson(dockerInfo, formatted = false)) + }, + executeCount, + containerHashId ).execute() } } @@ -195,6 +221,20 @@ class ThirdPartyAgentBuildDao { with(TDispatchThirdpartyAgentBuild.T_DISPATCH_THIRDPARTY_AGENT_BUILD) { return dslContext.selectFrom(this.forceIndex("IDX_AGENTID_STATUS_UPDATE")) .where(AGENT_ID.eq(agentId)) + .and(DOCKER_INFO.isNull) + .and(STATUS.`in`(PipelineTaskStatus.RUNNING.status, PipelineTaskStatus.QUEUE.status)) + .fetch() + } + } + + fun getDockerRunningAndQueueBuilds( + dslContext: DSLContext, + agentId: String + ): Result { + with(TDispatchThirdpartyAgentBuild.T_DISPATCH_THIRDPARTY_AGENT_BUILD) { + return dslContext.selectFrom(this.forceIndex("IDX_AGENTID_STATUS_UPDATE")) + .where(AGENT_ID.eq(agentId)) + .and(DOCKER_INFO.isNotNull) .and(STATUS.`in`(PipelineTaskStatus.RUNNING.status, PipelineTaskStatus.QUEUE.status)) .fetch() } diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentMonitorService.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentMonitorService.kt index 41ec8018634..175f023812e 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentMonitorService.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentMonitorService.kt @@ -50,6 +50,7 @@ import java.util.concurrent.TimeUnit /** * 三方构建机的业务监控拓展 */ +@Suppress("ALL") @Service class ThirdPartyAgentMonitorService @Autowired constructor( private val client: Client, @@ -59,7 +60,6 @@ class ThirdPartyAgentMonitorService @Autowired constructor( private val thirdPartyAgentBuildDao: ThirdPartyAgentBuildDao ) { - @Suppress("LongMethod", "NestedBlockDepth") fun monitor(event: AgentStartMonitor) { val record = thirdPartyAgentBuildDao.get(dslContext, event.buildId, event.vmSeqId) ?: return @@ -75,46 +75,78 @@ class ThirdPartyAgentMonitorService @Autowired constructor( val agentDetail = client.get(ServiceThirdPartyAgentResource::class) .getAgentDetail(userId = event.userId, projectId = event.projectId, agentHashId = record.agentId) - .data + .data ?: return - if (agentDetail != null) { - val tag = VMUtils.genStartVMTaskId(event.vmSeqId) - val heartbeatInfo = agentDetail.heartbeatInfo + val tag = VMUtils.genStartVMTaskId(event.vmSeqId) + val heartbeatInfo = agentDetail.heartbeatInfo - logMessage.append( - MessageCodeUtil.getCodeLanMessage( - messageCode = ProcessMessageCode.BUILD_AGENT_DETAIL_LINK_ERROR, - params = arrayOf(event.projectId, agentDetail.nodeId) - ) + logMessage.append( + MessageCodeUtil.getCodeLanMessage( + messageCode = ProcessMessageCode.BUILD_AGENT_DETAIL_LINK_ERROR, + params = arrayOf(event.projectId, agentDetail.nodeId) ) + ) + + // #7748 agent使用docker作为构建机 + var parallelTaskCount = agentDetail.parallelTaskCount + var busyTaskSize = heartbeatInfo?.busyTaskSize + if (record.dockerInfo != null) { + parallelTaskCount = agentDetail.dockerParallelTaskCount + busyTaskSize = heartbeatInfo?.dockerBusyTaskSize + } + + if (record.dockerInfo != null) { + logMessage.append("|Docker构建|最大并行构建量(maximum parallelism)/当前正在运行构建数量(Running): ") + } else { logMessage.append("|最大并行构建量(maximum parallelism)/当前正在运行构建数量(Running): ") - if (agentDetail.parallelTaskCount != "0") { - logMessage.append(agentDetail.parallelTaskCount).append("/").append(heartbeatInfo?.busyTaskSize ?: 0) - } + } + if (parallelTaskCount != "0") { + logMessage.append(parallelTaskCount).append("/") + .append(busyTaskSize ?: 0) + } - if (agentDetail.parallelTaskCount == "0") { - logMessage.append("无限制(unlimited), 注意负载(Attention)") - } - log(event, logMessage, tag) + if (parallelTaskCount == "0") { + logMessage.append("无限制(unlimited), 注意负载(Attention)") + } + log(event, logMessage, tag) - if (heartbeatInfo != null) { + if (heartbeatInfo == null) { + return + } - heartbeatInfo.heartbeatTime?.let { self -> - logMessage.append("构建机最近心跳时间（heartbeat Time): ${DateTimeUtil.formatDate(Date(self))}") - } + heartbeatInfo.heartbeatTime?.let { self -> + logMessage.append("构建机最近心跳时间（heartbeat Time): ${DateTimeUtil.formatDate(Date(self))}") + } - logMessage.append("|最近${heartbeatInfo.taskList.size}次运行中的构建:\n") + if (record.dockerInfo != null) { + logMessage.append("|Docker构建|最近${heartbeatInfo.dockerTaskList?.size ?: 0}次运行中的构建:\n") + } else { + logMessage.append("|最近${heartbeatInfo.taskList?.size ?: 0}次运行中的构建:\n") + } - heartbeatInfo.taskList.forEach { - thirdPartyAgentBuildDao.get(dslContext, it.buildId, it.vmSeqId)?.let { r1 -> - logMessage.append("") - logMessage.append("运行中(Running) #${r1.buildNum} (${r1.pipelineName} ${r1.taskName})\n") + if (record.dockerInfo != null) { + heartbeatInfo.dockerTaskList?.forEach dockerInfoFor@{ + thirdPartyAgentBuildDao.get(dslContext, it.buildId, it.vmSeqId)?.let { r1 -> + if (r1.dockerInfo == null) { + return@dockerInfoFor } + logMessage.append("") + logMessage.append("运行中(Running) #${r1.buildNum} (${r1.pipelineName} ${r1.taskName})\n") + } + } + } else { + heartbeatInfo.taskList?.forEach taskInfoFor@{ + thirdPartyAgentBuildDao.get(dslContext, it.buildId, it.vmSeqId)?.let { r1 -> + if (r1.dockerInfo != null) { + return@taskInfoFor + } + logMessage.append("") + logMessage.append("运行中(Running) #${r1.buildNum} (${r1.pipelineName} ${r1.taskName})\n") } - - log(event, logMessage, tag) } } + + log(event, logMessage, tag) } private fun log(event: AgentStartMonitor, sb: StringBuilder, tag: String) { @@ -130,11 +162,11 @@ class ThirdPartyAgentMonitorService @Autowired constructor( private fun genBuildDetailUrl(projectId: String, pipelineId: String, buildId: String): String { return HomeHostUtil.getHost(commonConfig.devopsHostGateway!!) + - "/console/pipeline/$projectId/$pipelineId/detail/$buildId" + "/console/pipeline/$projectId/$pipelineId/detail/$buildId" } /** - * 3分钟如果Agent端发起了构建任务领取后还没启动，尝试回退到队列让其以便能重新领取到。 + * 3分钟如果Agent端发起了构建任务领取后还没启动，尝试回退到队列让其以便能重新领取到。(docker 10min) * 用于解决Agent端几类极端问题场景： * 1、网络问题导致Agent侧领取中断，数据包丢失，没有收到领取任务，但任务已经被改成RUNNING，需要回退。 * 2、Agent领取后未处理构建前，进程意外退出 @@ -147,8 +179,13 @@ class ThirdPartyAgentMonitorService @Autowired constructor( } record.updatedTime?.let { self -> + val outTime = if (record.dockerInfo != null) { + System.currentTimeMillis() - self.timestampmilli() > TimeUnit.MINUTES.toMillis(DOCKER_ROLLBACK_MIN) + } else { + System.currentTimeMillis() - self.timestampmilli() > TimeUnit.MINUTES.toMillis(ROLLBACK_MIN) + } // Agent发起领取超过x分钟没有启动，基本上存在问题需要重回队列以便被再次调度到 - if (System.currentTimeMillis() - self.timestampmilli() > TimeUnit.MINUTES.toMillis(ROLLBACK_MIN)) { + if (outTime) { thirdPartyAgentBuildDao.updateStatus(dslContext, record.id, PipelineTaskStatus.QUEUE) sb.append("任务领取超过$ROLLBACK_MIN 分钟没有启动, 可能存在异常，开始重置") .append("(Over $ROLLBACK_MIN minutes, try roll back to queue.)") @@ -159,5 +196,6 @@ class ThirdPartyAgentMonitorService @Autowired constructor( companion object { private const val ROLLBACK_MIN = 3L // 3分钟如果构建任务领取后没启动，尝试回退状态 + private const val DOCKER_ROLLBACK_MIN = 10L // 针对docker构建场景增加拉镜像可能需要的时间 } } diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentService.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentService.kt index 1987c99f327..767e06766c3 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentService.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentService.kt @@ -27,6 +27,7 @@ package com.tencent.devops.dispatch.service +import com.fasterxml.jackson.core.type.TypeReference import com.tencent.devops.common.api.enums.AgentStatus import com.tencent.devops.common.api.exception.OperationException import com.tencent.devops.common.api.exception.RemoteServiceException @@ -35,9 +36,11 @@ import com.tencent.devops.common.api.pojo.Page import com.tencent.devops.common.api.pojo.SimpleResult import com.tencent.devops.common.api.pojo.agent.UpgradeItem import com.tencent.devops.common.api.util.HashUtil +import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.PageUtil import com.tencent.devops.common.api.util.timestamp import com.tencent.devops.common.client.Client +import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentDockerInfoDispatch import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.dispatch.dao.ThirdPartyAgentBuildDao import com.tencent.devops.dispatch.pojo.ThirdPartyAgentPreBuildAgents @@ -45,6 +48,7 @@ import com.tencent.devops.dispatch.pojo.enums.PipelineTaskStatus import com.tencent.devops.dispatch.pojo.thirdPartyAgent.AgentBuildInfo import com.tencent.devops.dispatch.pojo.thirdPartyAgent.ThirdPartyBuildInfo import com.tencent.devops.dispatch.pojo.thirdPartyAgent.ThirdPartyBuildWithStatus +import com.tencent.devops.dispatch.service.dispatcher.agent.DispatchService import com.tencent.devops.dispatch.utils.ThirdPartyAgentLock import com.tencent.devops.dispatch.utils.redis.ThirdPartyAgentBuildRedisUtils import com.tencent.devops.environment.api.thirdPartyAgent.ServiceThirdPartyAgentResource @@ -52,6 +56,7 @@ import com.tencent.devops.environment.pojo.thirdPartyAgent.ThirdPartyAgent import com.tencent.devops.environment.pojo.thirdPartyAgent.ThirdPartyAgentUpgradeByVersionInfo import com.tencent.devops.model.dispatch.tables.records.TDispatchThirdpartyAgentBuildRecord import com.tencent.devops.process.api.service.ServiceBuildResource +import com.tencent.devops.process.pojo.mq.PipelineAgentShutdownEvent import com.tencent.devops.process.pojo.mq.PipelineAgentStartupEvent import org.jooq.DSLContext import org.slf4j.LoggerFactory @@ -67,18 +72,20 @@ class ThirdPartyAgentService @Autowired constructor( private val thirdPartyAgentBuildRedisUtils: ThirdPartyAgentBuildRedisUtils, private val client: Client, private val redisOperation: RedisOperation, - private val thirdPartyAgentBuildDao: ThirdPartyAgentBuildDao + private val thirdPartyAgentBuildDao: ThirdPartyAgentBuildDao, + private val dispatchService: DispatchService ) { fun queueBuild( agent: ThirdPartyAgent, thirdPartyAgentWorkspace: String, event: PipelineAgentStartupEvent, - retryCount: Int = 0 + retryCount: Int = 0, + dockerInfo: ThirdPartyAgentDockerInfoDispatch? ) { with(event) { try { - val count = thirdPartyAgentBuildDao.add( + thirdPartyAgentBuildDao.add( dslContext = dslContext, projectId = projectId, agentId = agent.agentId, @@ -90,12 +97,15 @@ class ThirdPartyAgentService @Autowired constructor( buildNum = buildNo, taskName = taskName, agentIp = agent.ip, - nodeId = HashUtil.decodeIdToLong(agent.nodeId ?: "") + nodeId = HashUtil.decodeIdToLong(agent.nodeId ?: ""), + dockerInfo = dockerInfo, + executeCount = event.executeCount, + containerHashId = event.containerHashId ) } catch (e: DeadlockLoserDataAccessException) { logger.warn("Fail to add the third party agent build of ($buildId|$vmSeqId|${agent.agentId}") if (retryCount <= QUEUE_RETRY_COUNT) { - queueBuild(agent, thirdPartyAgentWorkspace, event) + queueBuild(agent, thirdPartyAgentWorkspace, event, retryCount + 1, dockerInfo) } else { throw OperationException("Fail to add the third party agent build") } @@ -123,6 +133,10 @@ class ThirdPartyAgentService @Autowired constructor( return thirdPartyAgentBuildDao.getRunningAndQueueBuilds(dslContext, agentId).size } + fun getDockerRunningBuilds(agentId: String): Int { + return thirdPartyAgentBuildDao.getDockerRunningAndQueueBuilds(dslContext, agentId).size + } + fun startBuild(projectId: String, agentId: String, secretKey: String): AgentResult { // Get the queue status build by buildId and agentId logger.debug("Start the third party agent($agentId) of project($projectId)") @@ -151,7 +165,7 @@ class ThirdPartyAgentService @Autowired constructor( if (agentResult.data!!.secretKey != secretKey) { logger.warn( "The secretKey($secretKey) is not match the expect one(${agentResult.data!!.secretKey} " + - "of project($projectId) and agent($agentId)" + "of project($projectId) and agent($agentId)" ) throw NotFoundException("Fail to get the agent") } @@ -171,20 +185,48 @@ class ThirdPartyAgentService @Autowired constructor( return AgentResult(AgentStatus.IMPORT_OK, null) } + logger.debug( + "Third party agent($agentId) start up agent project($projectId) build project(${build.projectId})" + ) + logger.info("Start the build(${build.buildId}) of agent($agentId) and seq(${build.vmSeqId})") thirdPartyAgentBuildDao.updateStatus(dslContext, build.id, PipelineTaskStatus.RUNNING) try { client.get(ServiceThirdPartyAgentResource::class) - .agentTaskStarted(projectId, build.pipelineId, build.buildId, build.vmSeqId, build.agentId) + .agentTaskStarted( + build.projectId, + build.pipelineId, + build.buildId, + build.vmSeqId, + build.agentId + ) } catch (e: RemoteServiceException) { - logger.warn("notify agent task[$projectId|${build.buildId}|${build.vmSeqId}|$agentId]" + - " claim failed, cause: ${e.message}") + logger.warn( + "notify agent task[$build.projectId|${build.buildId}|${build.vmSeqId}|$agentId]" + + " claim failed, cause: ${e.message} agent project($projectId)" + ) } return AgentResult( AgentStatus.IMPORT_OK, - ThirdPartyBuildInfo(projectId, build.buildId, build.vmSeqId, build.workspace) + ThirdPartyBuildInfo( + projectId = build.projectId, + buildId = build.buildId, + vmSeqId = build.vmSeqId, + workspace = build.workspace, + pipelineId = build.pipelineId, + dockerBuildInfo = if (build.dockerInfo == null) { + null + } else { + JsonUtil.getObjectMapper().readValue( + build.dockerInfo.data(), + object : TypeReference() {} + ) + }, + executeCount = build.executeCount, + containerHashId = build.containerHashId + ) ) } finally { redisLock.unlock() @@ -238,7 +280,14 @@ class ThirdPartyAgentService @Autowired constructor( } } catch (t: Throwable) { logger.warn("Fail to check if agent can upgrade", t) - AgentResult(AgentStatus.IMPORT_EXCEPTION, UpgradeItem(false, false, false)) + AgentResult( + AgentStatus.IMPORT_EXCEPTION, UpgradeItem( + agent = false, + worker = false, + jdk = false, + dockerInitFile = false + ) + ) } } @@ -273,7 +322,10 @@ class ThirdPartyAgentService @Autowired constructor( } } - fun finishBuild(buildId: String, vmSeqId: String?, success: Boolean) { + fun finishBuild(event: PipelineAgentShutdownEvent) { + val buildId = event.buildId + val vmSeqId = event.vmSeqId + val success = event.buildResult if (vmSeqId.isNullOrBlank()) { val records = thirdPartyAgentBuildDao.list(dslContext, buildId) if (records.isEmpty()) { @@ -281,10 +333,18 @@ class ThirdPartyAgentService @Autowired constructor( } records.forEach { finishBuild(it, success) + if (it.dockerInfo != null) { + // 第三方构建机可能是docker构建机时需要在这里删除docker类型的redisKey + dispatchService.shutdown(event) + } } } else { val record = thirdPartyAgentBuildDao.get(dslContext, buildId, vmSeqId) ?: return finishBuild(record, success) + if (record.dockerInfo != null) { + // 第三方构建机可能是docker构建机时需要在这里删除docker类型的redisKey + dispatchService.shutdown(event) + } } } @@ -318,7 +378,7 @@ class ThirdPartyAgentService @Autowired constructor( private fun finishBuild(record: TDispatchThirdpartyAgentBuildRecord, success: Boolean) { logger.info( "Finish the third party agent(${record.agentId}) build(${record.buildId}) " + - "of seq(${record.vmSeqId}) and status(${record.status})" + "of seq(${record.vmSeqId}) and status(${record.status})" ) val agentResult = client.get(ServiceThirdPartyAgentResource::class) .getAgentById(record.projectId, record.agentId) @@ -367,17 +427,25 @@ class ThirdPartyAgentService @Autowired constructor( thirdPartyAgentBuildDao.updateStatus( dslContext = dslContext, id = buildRecord.id, - status = PipelineTaskStatus.DONE + status = if (!buildInfo.success) { + PipelineTaskStatus.FAILURE + } else { + PipelineTaskStatus.DONE + } ) } client.get(ServiceBuildResource::class).workerBuildFinish( - projectId = projectId, + projectId = buildInfo.projectId, pipelineId = if (buildInfo.pipelineId.isNullOrBlank()) "dummyPipelineId" else buildInfo.pipelineId!!, buildId = buildInfo.buildId, vmSeqId = buildInfo.vmSeqId, nodeHashId = agentResult.data!!.nodeId, - simpleResult = SimpleResult(success = buildInfo.success, message = buildInfo.message) + simpleResult = SimpleResult( + success = buildInfo.success, + message = buildInfo.message, + error = buildInfo.error + ) ) } diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/DispatchService.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/DispatchService.kt new file mode 100644 index 00000000000..0b3030d08ef --- /dev/null +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/DispatchService.kt @@ -0,0 +1,105 @@ +package com.tencent.devops.dispatch.service.dispatcher.agent + +import com.fasterxml.jackson.databind.ObjectMapper +import com.tencent.devops.common.api.util.ApiUtil +import com.tencent.devops.common.api.util.HashUtil +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.process.pojo.mq.PipelineAgentShutdownEvent +import com.tencent.devops.process.pojo.mq.PipelineAgentStartupEvent +import java.util.Date +import java.util.concurrent.TimeUnit +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +/** issue_7748 搬用 dispatch sdk 的方法，因为sdk集成当前存在问题 + * @see com.tencent.devops.common.dispatch.sdk.service.DispatchService + **/ +@Suppress("ALL") +@Service +class DispatchService @Autowired constructor( + private val redisOperation: RedisOperation, + private val objectMapper: ObjectMapper +) { + + fun setRedisAuth(event: PipelineAgentStartupEvent): SecretInfo { + val secretInfoRedisKey = secretInfoRedisKey(event.buildId) + val redisResult = redisOperation.hget( + key = secretInfoRedisKey, + hashKey = secretInfoRedisMapKey(event.vmSeqId, event.executeCount ?: 1) + ) + if (redisResult != null) { + return JsonUtil.to(redisResult, SecretInfo::class.java) + } + val secretKey = ApiUtil.randomSecretKey() + val hashId = HashUtil.encodeLongId(System.currentTimeMillis()) + logger.info("[${event.buildId}|${event.vmSeqId}] Start to build the event with ($hashId|$secretKey)") + redisOperation.set( + key = redisKey(hashId, secretKey), + value = objectMapper.writeValueAsString( + RedisBuild( + vmName = event.vmNames.ifBlank { "Dispatcher-sdk-${event.vmSeqId}" }, + projectId = event.projectId, + pipelineId = event.pipelineId, + buildId = event.buildId, + vmSeqId = event.vmSeqId, + channelCode = event.channelCode, + zone = event.zone, + atoms = event.atoms, + executeCount = event.executeCount ?: 1 + ) + ), + expiredInSecond = TimeUnit.DAYS.toSeconds(7) + ) + + // 一周过期时间 + redisOperation.hset( + secretInfoRedisKey(event.buildId), + secretInfoRedisMapKey(event.vmSeqId, event.executeCount ?: 1), + JsonUtil.toJson(SecretInfo(hashId, secretKey), formatted = false) + ) + val expireAt = System.currentTimeMillis() + TimeUnit.DAYS.toMillis(7) + redisOperation.expireAt(secretInfoRedisKey, Date(expireAt)) + return SecretInfo( + hashId = hashId, + secretKey = secretKey + ) + } + + private fun redisKey(hashId: String, secretKey: String) = + "docker_build_key_${hashId}_$secretKey" + + private fun secretInfoRedisKey(buildId: String) = + "secret_info_key_$buildId" + + private fun secretInfoRedisMapKey(vmSeqId: String, executeCount: Int) = "$vmSeqId-$executeCount" + + fun shutdown(event: PipelineAgentShutdownEvent) { + val secretInfoKey = secretInfoRedisKey(event.buildId) + + // job结束 + finishBuild(event.vmSeqId!!, event.buildId, event.executeCount ?: 1) + redisOperation.hdelete(secretInfoKey, secretInfoRedisMapKey(event.vmSeqId!!, event.executeCount ?: 1)) + + val keysSet = redisOperation.hkeys(secretInfoKey) + if (keysSet.isNullOrEmpty()) { + redisOperation.delete(secretInfoKey) + } + } + + private fun finishBuild(vmSeqId: String, buildId: String, executeCount: Int) { + val result = redisOperation.hget(secretInfoRedisKey(buildId), secretInfoRedisMapKey(vmSeqId, executeCount)) + if (result != null) { + val secretInfo = JsonUtil.to(result, SecretInfo::class.java) + redisOperation.delete(redisKey(secretInfo.hashId, secretInfo.secretKey)) + logger.warn("$buildId|$vmSeqId finishBuild success.") + } else { + logger.warn("$buildId|$vmSeqId finishBuild failed, secretInfo is null.") + } + } + + companion object { + private val logger = LoggerFactory.getLogger(DispatchService::class.java) + } +} diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/RedisBuild.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/RedisBuild.kt new file mode 100644 index 00000000000..b43ec9d38b0 --- /dev/null +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/RedisBuild.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.service.dispatcher.agent + +import com.tencent.devops.common.api.pojo.Zone + +/** issue_7748 搬用 dispatch sdk 的类，因为sdk集成当前存在问题 + * @see com.tencent.devops.common.dispatch.sdk.pojo.RedisBuild + **/ +data class RedisBuild( + val vmName: String, + val projectId: String, + val pipelineId: String, + val buildId: String, + val vmSeqId: String, + val channelCode: String?, + val zone: Zone?, + val atoms: Map = mapOf(), // 用插件框架开发的插件信息 key为插件code，value为下载路径 + val executeCount: Int? = 1 +) diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/SecretInfo.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/SecretInfo.kt new file mode 100644 index 00000000000..7e3d9433b52 --- /dev/null +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/SecretInfo.kt @@ -0,0 +1,9 @@ +package com.tencent.devops.dispatch.service.dispatcher.agent + +/** issue_7748 搬用 dispatch sdk 的类，因为sdk集成当前存在问题 + * @see com.tencent.devops.common.dispatch.sdk.pojo.SecretInfo + **/ +data class SecretInfo( + val hashId: String, + val secretKey: String +) diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/ThirdPartyAgentDispatcher.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/ThirdPartyAgentDispatcher.kt index 6b080de8e5a..2f8838184bb 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/ThirdPartyAgentDispatcher.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/ThirdPartyAgentDispatcher.kt @@ -35,6 +35,8 @@ import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatch import com.tencent.devops.common.log.utils.BuildLogPrinter import com.tencent.devops.common.pipeline.enums.VMBaseOS import com.tencent.devops.common.pipeline.type.agent.AgentType +import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentDockerInfo +import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentDockerInfoDispatch import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentEnvDispatchType import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentIDDispatchType import com.tencent.devops.common.pipeline.type.agent.ThirdPartyDevCloudDispatchType @@ -61,14 +63,15 @@ import org.springframework.stereotype.Component import javax.ws.rs.core.Response @Component -@Suppress("NestedBlockDepth") +@Suppress("ALL") class ThirdPartyAgentDispatcher @Autowired constructor( private val client: Client, private val redisOperation: RedisOperation, private val buildLogPrinter: BuildLogPrinter, private val thirdPartyAgentBuildRedisUtils: ThirdPartyAgentBuildRedisUtils, private val pipelineEventDispatcher: PipelineEventDispatcher, - private val thirdPartyAgentBuildService: ThirdPartyAgentService + private val thirdPartyAgentBuildService: ThirdPartyAgentService, + private val dispatchService: DispatchService ) : Dispatcher { override fun canDispatch(event: PipelineAgentStartupEvent) = event.dispatchType is ThirdPartyAgentIDDispatchType || @@ -81,6 +84,7 @@ class ThirdPartyAgentDispatcher @Autowired constructor( val dispatchType = event.dispatchType as ThirdPartyAgentIDDispatchType buildByAgentId(event, dispatchType) } + is ThirdPartyDevCloudDispatchType -> { val originDispatchType = event.dispatchType as ThirdPartyDevCloudDispatchType buildByAgentId( @@ -88,14 +92,17 @@ class ThirdPartyAgentDispatcher @Autowired constructor( dispatchType = ThirdPartyAgentIDDispatchType( displayName = originDispatchType.displayName, workspace = originDispatchType.workspace, - agentType = originDispatchType.agentType + agentType = originDispatchType.agentType, + dockerInfo = null ) ) } + is ThirdPartyAgentEnvDispatchType -> { val dispatchType = event.dispatchType as ThirdPartyAgentEnvDispatchType buildByEnvId(event, dispatchType) } + else -> { throw InvalidParamException("Unknown agent type - ${event.dispatchType}") } @@ -104,11 +111,7 @@ class ThirdPartyAgentDispatcher @Autowired constructor( override fun shutdown(event: PipelineAgentShutdownEvent) { try { - thirdPartyAgentBuildService.finishBuild( - buildId = event.buildId, - vmSeqId = event.vmSeqId, - success = event.buildResult - ) + thirdPartyAgentBuildService.finishBuild(event) } finally { try { sendDispatchMonitoring( @@ -179,7 +182,7 @@ class ThirdPartyAgentDispatcher @Autowired constructor( return } - if (!buildByAgentId(event, agentResult.data!!, dispatchType.workspace)) { + if (!buildByAgentId(event, agentResult.data!!, dispatchType.workspace, dispatchType.dockerInfo)) { retry( client = client, buildLogPrinter = buildLogPrinter, @@ -212,7 +215,12 @@ class ThirdPartyAgentDispatcher @Autowired constructor( } } - private fun buildByAgentId(event: PipelineAgentStartupEvent, agent: ThirdPartyAgent, workspace: String?): Boolean { + private fun buildByAgentId( + event: PipelineAgentStartupEvent, + agent: ThirdPartyAgent, + workspace: String?, + dockerInfo: ThirdPartyAgentDockerInfo? + ): Boolean { val redisLock = ThirdPartyAgentLock(redisOperation, event.projectId, agent.agentId) try { if (redisLock.tryLock()) { @@ -222,8 +230,30 @@ class ThirdPartyAgentDispatcher @Autowired constructor( return false } + // 生成docker构建机类型的id和secretKey + val message = if (dockerInfo == null) { + null + } else { + dispatchService.setRedisAuth(event) + } + // #5806 入库失败就不再写Redis - inQueue(agent = agent, event = event, agentId = agent.agentId, workspace = workspace) + inQueue( + agent = agent, + event = event, + agentId = agent.agentId, + workspace = workspace, + dockerInfo = if (dockerInfo == null) { + null + } else { + ThirdPartyAgentDockerInfoDispatch( + agentId = message!!.hashId, + secretKey = message.secretKey, + info = dockerInfo + ) + } + ) + // 保存构建详情 saveAgentInfoToBuildDetail(event = event, agent = agent) @@ -249,12 +279,19 @@ class ThirdPartyAgentDispatcher @Autowired constructor( ) } - private fun inQueue(agent: ThirdPartyAgent, event: PipelineAgentStartupEvent, agentId: String, workspace: String?) { - + private fun inQueue( + agent: ThirdPartyAgent, + event: PipelineAgentStartupEvent, + agentId: String, + workspace: String?, + dockerInfo: ThirdPartyAgentDockerInfoDispatch? + ) { thirdPartyAgentBuildService.queueBuild( agent = agent, thirdPartyAgentWorkspace = workspace ?: "", - event = event + event = event, + retryCount = 0, + dockerInfo = dockerInfo ) thirdPartyAgentBuildRedisUtils.setThirdPartyBuild( @@ -337,7 +374,7 @@ class ThirdPartyAgentDispatcher @Autowired constructor( if (agentsResult.isNotOk()) { logger.warn( "${event.buildId}|START_AGENT_FAILED|" + - "j(${event.vmSeqId})|dispatchType=$dispatchType|err=${agentsResult.message}" + "j(${event.vmSeqId})|dispatchType=$dispatchType|err=${agentsResult.message}" ) retry( client = client, @@ -411,6 +448,8 @@ class ThirdPartyAgentDispatcher @Autowired constructor( val hasTryAgents = HashSet() val runningBuildsMapper = HashMap() + // docker和二进制任务区分开，所以单独设立一个 + val dockerRunningBuildsMapper = HashMap() /** * 1. 最高优先级的agent: @@ -440,7 +479,8 @@ class ThirdPartyAgentDispatcher @Autowired constructor( dispatchType = dispatchType, agents = preBuildAgents, hasTryAgents = hasTryAgents, - runningBuildsMapper = runningBuildsMapper + runningBuildsMapper = runningBuildsMapper, + dockerRunningBuildsMapper = dockerRunningBuildsMapper ) ) { logger.info( @@ -467,7 +507,8 @@ class ThirdPartyAgentDispatcher @Autowired constructor( dispatchType = dispatchType, agents = preBuildAgents, hasTryAgents = hasTryAgents, - runningBuildsMapper = runningBuildsMapper + runningBuildsMapper = runningBuildsMapper, + dockerRunningBuildsMapper = dockerRunningBuildsMapper ) ) { logger.info( @@ -489,7 +530,8 @@ class ThirdPartyAgentDispatcher @Autowired constructor( dispatchType = dispatchType, agents = activeAgents, hasTryAgents = hasTryAgents, - runningBuildsMapper = runningBuildsMapper + runningBuildsMapper = runningBuildsMapper, + dockerRunningBuildsMapper = dockerRunningBuildsMapper ) ) { logger.info( @@ -511,7 +553,8 @@ class ThirdPartyAgentDispatcher @Autowired constructor( dispatchType = dispatchType, agents = activeAgents, hasTryAgents = hasTryAgents, - runningBuildsMapper = runningBuildsMapper + runningBuildsMapper = runningBuildsMapper, + dockerRunningBuildsMapper = dockerRunningBuildsMapper ) ) { logger.info( @@ -582,7 +625,8 @@ class ThirdPartyAgentDispatcher @Autowired constructor( dispatchType: ThirdPartyAgentEnvDispatchType, agents: HashSet, hasTryAgents: HashSet, - runningBuildsMapper: HashMap + runningBuildsMapper: HashMap, + dockerRunningBuildsMapper: HashMap ): Boolean { return startAgentsForEnvBuild( event = event, @@ -590,8 +634,17 @@ class ThirdPartyAgentDispatcher @Autowired constructor( agents = agents, hasTryAgents = hasTryAgents, runningBuildsMapper = runningBuildsMapper, + dockerRunningBuildsMapper = dockerRunningBuildsMapper, agentMatcher = object : AgentMatcher { - override fun match(runningCnt: Int, agent: ThirdPartyAgent): Boolean { + override fun match( + runningCnt: Int, + agent: ThirdPartyAgent, + dockerBuilder: Boolean, + dockerRunningCnt: Int + ): Boolean { + if (dockerBuilder) { + return dockerRunningCnt == 0 + } return runningCnt == 0 } } @@ -603,7 +656,8 @@ class ThirdPartyAgentDispatcher @Autowired constructor( dispatchType: ThirdPartyAgentEnvDispatchType, agents: HashSet, hasTryAgents: HashSet, - runningBuildsMapper: HashMap + runningBuildsMapper: HashMap, + dockerRunningBuildsMapper: HashMap ): Boolean { return startAgentsForEnvBuild( event = event, @@ -611,8 +665,23 @@ class ThirdPartyAgentDispatcher @Autowired constructor( agents = agents, hasTryAgents = hasTryAgents, runningBuildsMapper = runningBuildsMapper, + dockerRunningBuildsMapper = dockerRunningBuildsMapper, agentMatcher = object : AgentMatcher { - override fun match(runningCnt: Int, agent: ThirdPartyAgent): Boolean { + override fun match( + runningCnt: Int, + agent: ThirdPartyAgent, + dockerBuilder: Boolean, + dockerRunningCnt: Int + ): Boolean { + if (dockerBuilder) { + if (agent.dockerParallelTaskCount != null && + agent.dockerParallelTaskCount!! > 0 && + agent.dockerParallelTaskCount!! > dockerRunningCnt + ) { + return true + } + return false + } if (agent.parallelTaskCount != null && agent.parallelTaskCount!! > 0 && agent.parallelTaskCount!! > runningCnt @@ -631,6 +700,7 @@ class ThirdPartyAgentDispatcher @Autowired constructor( agents: HashSet, hasTryAgents: HashSet, runningBuildsMapper: HashMap, + dockerRunningBuildsMapper: HashMap, agentMatcher: AgentMatcher ): Boolean { if (agents.isNotEmpty()) { @@ -639,7 +709,18 @@ class ThirdPartyAgentDispatcher @Autowired constructor( return@forEach } val runningCnt = getRunningCnt(it.agentId, runningBuildsMapper) - if (agentMatcher.match(runningCnt, it)) { + val dockerRunningCnt = if (dispatchType.dockerInfo == null) { + 0 + } else { + getDockerRunningCnt(it.agentId, dockerRunningBuildsMapper) + } + if (agentMatcher.match( + runningCnt = runningCnt, + agent = it, + dockerBuilder = dispatchType.dockerInfo != null, + dockerRunningCnt = dockerRunningCnt + ) + ) { if (startEnvAgentBuild(event, it, dispatchType, hasTryAgents)) { logger.info( "[${it.projectId}|$[${event.pipelineId}|${event.buildId}|${it.agentId}] " + @@ -663,7 +744,7 @@ class ThirdPartyAgentDispatcher @Autowired constructor( return false } hasTryAgents.add(agent.agentId) - if (buildByAgentId(event, agent, dispatchType.workspace)) { + if (buildByAgentId(event, agent, dispatchType.workspace, dispatchType.dockerInfo)) { return true } return false @@ -678,8 +759,17 @@ class ThirdPartyAgentDispatcher @Autowired constructor( return runningCnt } + private fun getDockerRunningCnt(agentId: String, dockerRunningBuildsMapper: HashMap): Int { + var dockerRunningCnt = dockerRunningBuildsMapper[agentId] + if (dockerRunningCnt == null) { + dockerRunningCnt = thirdPartyAgentBuildService.getDockerRunningBuilds(agentId) + dockerRunningBuildsMapper[agentId] = dockerRunningCnt + } + return dockerRunningCnt + } + interface AgentMatcher { - fun match(runningCnt: Int, agent: ThirdPartyAgent): Boolean + fun match(runningCnt: Int, agent: ThirdPartyAgent, dockerBuilder: Boolean, dockerRunningCnt: Int): Boolean } companion object { diff --git a/src/backend/ci/core/dispatch/boot-dispatch/build.gradle.kts b/src/backend/ci/core/dispatch/boot-dispatch/build.gradle.kts index 879cfc0164c..af9a5573d19 100644 --- a/src/backend/ci/core/dispatch/boot-dispatch/build.gradle.kts +++ b/src/backend/ci/core/dispatch/boot-dispatch/build.gradle.kts @@ -24,7 +24,6 @@ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ - dependencies { api(project(":core:dispatch:biz-dispatch-sample")) api(project(":core:common:common-auth:common-auth-mock")) diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeBaseInfo.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeBaseInfo.kt index fa15a829424..67f733da445 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeBaseInfo.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeBaseInfo.kt @@ -30,7 +30,7 @@ package com.tencent.devops.environment.pojo import io.swagger.annotations.ApiModel import io.swagger.annotations.ApiModelProperty -@ApiModel("节点信息(权限)") +@ApiModel("NodeBaseInfo-节点信息(权限)") data class NodeBaseInfo( @ApiModelProperty("环境 HashId", required = true) val nodeHashId: String, diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeWithPermission.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeWithPermission.kt index b061c167fc5..81630923e38 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeWithPermission.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeWithPermission.kt @@ -30,7 +30,7 @@ package com.tencent.devops.environment.pojo import io.swagger.annotations.ApiModel import io.swagger.annotations.ApiModelProperty -@ApiModel("节点信息(权限)") +@ApiModel("NodeWithPermission-节点信息(权限)") data class NodeWithPermission( @ApiModelProperty("环境 HashId", required = true) val nodeHashId: String, diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/HeartbeatResponse.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/HeartbeatResponse.kt index a633fd86f56..35d73cf4bfb 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/HeartbeatResponse.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/HeartbeatResponse.kt @@ -48,5 +48,7 @@ data class HeartbeatResponse( @ApiModelProperty("文件网关路径") val fileGateway: String? = "", @ApiModelProperty("Agent的一些属性配置") - val props: Map + val props: Map, + @ApiModelProperty("docker最大任务数量") + val dockerParallelTaskCount: Int ) diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgent.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgent.kt index eaf1f82db33..d7dbc396179 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgent.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgent.kt @@ -54,5 +54,7 @@ data class ThirdPartyAgent( @ApiModelProperty("创建时间", required = true) val createTime: Long, @ApiModelProperty("并行执行的个数", required = false) - val parallelTaskCount: Int? = 4 + val parallelTaskCount: Int? = 4, + @ApiModelProperty("Docker构建机并行执行的个数", required = false) + val dockerParallelTaskCount: Int? = 4 ) diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgentDetail.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgentDetail.kt index dcf3676db77..58d1a1142a2 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgentDetail.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgentDetail.kt @@ -65,6 +65,8 @@ data class ThirdPartyAgentDetail( val maxParallelTaskCount: String, @ApiModelProperty("通道数量", required = true) val parallelTaskCount: String, + @ApiModelProperty("docker构建机通道数量", required = true) + val dockerParallelTaskCount: String, @ApiModelProperty("启动用户", required = true) val startedUser: String, @ApiModelProperty("agent链接", required = true) diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgentUpgradeByVersionInfo.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgentUpgradeByVersionInfo.kt index 1107fc2178e..ee3d8c66206 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgentUpgradeByVersionInfo.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdPartyAgent/ThirdPartyAgentUpgradeByVersionInfo.kt @@ -1,5 +1,6 @@ package com.tencent.devops.environment.pojo.thirdPartyAgent +import com.tencent.devops.common.api.pojo.agent.DockerInitFileInfo import io.swagger.annotations.ApiModel import io.swagger.annotations.ApiModelProperty @@ -10,5 +11,7 @@ data class ThirdPartyAgentUpgradeByVersionInfo( @ApiModelProperty("go agent 版本") val goAgentVersion: String?, @ApiModelProperty("jdk版本") - val jdkVersion: List? + val jdkVersion: List?, + @ApiModelProperty("docker init 文件升级信息") + val dockerInitFileInfo: DockerInitFileInfo? ) diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/model/AgentProps.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/model/AgentProps.kt index 6cbc3462aa0..0ccc0aec827 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/model/AgentProps.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/model/AgentProps.kt @@ -1,12 +1,16 @@ package com.tencent.devops.environment.model +import com.tencent.devops.common.api.pojo.agent.DockerInitFileInfo + /** * Agent 系统属性 * @see com.tencent.devops.environment.model arch 系统架构 * @param jdkVersion jdk版本 + * @param dockerInitFileInfo dockerInit文件信息 */ data class AgentProps( val arch: String, val jdkVersion: List, - val userProps: Map? + val userProps: Map?, + val dockerInitFileInfo: DockerInitFileInfo? ) diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/BluekingAgentUrlServiceImpl.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/BluekingAgentUrlServiceImpl.kt index 3533b61c7b7..c4cc7c2b26c 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/BluekingAgentUrlServiceImpl.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/BluekingAgentUrlServiceImpl.kt @@ -49,7 +49,12 @@ class BluekingAgentUrlServiceImpl constructor( override fun genAgentUrl(agentRecord: TEnvironmentThirdpartyAgentRecord): String { val gw = genGateway(agentRecord) val agentHashId = HashUtil.encodeLongId(agentRecord.id) - return "$gw/ms/environment/api/external/thirdPartyAgent/$agentHashId/agent?arch=\${ARCH}" + return if (agentRecord.os == OS.WINDOWS.name) { + // windows下不需要区分架构，删除arch + "$gw/ms/environment/api/external/thirdPartyAgent/$agentHashId/agent" + } else { + "$gw/ms/environment/api/external/thirdPartyAgent/$agentHashId/agent?arch=\${ARCH}" + } } override fun genAgentInstallScript(agentRecord: TEnvironmentThirdpartyAgentRecord): String { diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/AgentUpgradeService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/AgentUpgradeService.kt index 9639029b7fb..e3e2c04b90b 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/AgentUpgradeService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/AgentUpgradeService.kt @@ -72,21 +72,17 @@ class AgentUpgradeService @Autowired constructor( private fun listCanUpdateAgents( maxParallelCount: Int ): List? { - val currentVersion = redisOperation.get( - key = agentGrayUtils.getAgentVersionKey(), - isDistinguishCluster = true - )?.ifBlank { + val currentVersion = upgradeService.getWorkerVersion().ifBlank { logger.warn("invalid server agent version") return null - } ?: return null + } - val currentMasterVersion = redisOperation.get( - key = agentGrayUtils.getAgentMasterVersionKey(), - isDistinguishCluster = true - )?.ifBlank { + val currentMasterVersion = upgradeService.getAgentVersion().ifBlank { logger.warn("invalid server agent version") return null - } ?: return null + } + + val currentDockerInitFileMd5 = upgradeService.getDockerInitFileMd5() val importOKAgents = thirdPartyAgentDao.listByStatus( dslContext = dslContext, @@ -99,6 +95,7 @@ class AgentUpgradeService @Autowired constructor( checkProjectRouter(it.projectId) -> checkCanUpgrade( goAgentCurrentVersion = currentMasterVersion, workCurrentVersion = currentVersion, + currentDockerInitFileMd5 = currentDockerInitFileMd5, record = it ) @@ -116,31 +113,44 @@ class AgentUpgradeService @Autowired constructor( return client.get(ServiceProjectTagResource::class).checkProjectRouter(projectId).data ?: false } + @Suppress("ComplexMethod") private fun checkCanUpgrade( goAgentCurrentVersion: String, workCurrentVersion: String, + currentDockerInitFileMd5: String, record: TEnvironmentThirdpartyAgentRecord ): Boolean { AgentUpgradeType.values().forEach { type -> - var res = false - when (type) { + val res = when (type) { AgentUpgradeType.GO_AGENT -> { - res = goAgentCurrentVersion.trim() != record.masterVersion.trim() + goAgentCurrentVersion.trim() != record.masterVersion.trim() } AgentUpgradeType.WORKER -> { - res = workCurrentVersion.trim() != record.version.trim() + workCurrentVersion.trim() != record.version.trim() } AgentUpgradeType.JDK -> { val props = upgradeService.parseAgentProps(record.agentProps) ?: return@forEach val currentJdkVersion = upgradeService.getJdkVersion(record.os, props.arch) ?: return@forEach - res = if (props.jdkVersion.size > 2) { + if (props.jdkVersion.size > 2) { currentJdkVersion.trim() != props.jdkVersion.last().trim() } else { false } } + + AgentUpgradeType.DOCKER_INIT_FILE -> { + if (currentDockerInitFileMd5.isBlank()) { + return@forEach + } + val props = upgradeService.parseAgentProps(record.agentProps) ?: return@forEach + if (props.dockerInitFileInfo?.needUpgrade != true) { + return@forEach + } + (props.dockerInitFileInfo.fileMd5.isNotBlank() && + props.dockerInitFileInfo.fileMd5.trim() != currentDockerInitFileMd5.trim()) + } } if (res) { return true diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/ThirdPartyAgentMgrService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/ThirdPartyAgentMgrService.kt index 56b26940892..490f6640725 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/ThirdPartyAgentMgrService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/ThirdPartyAgentMgrService.kt @@ -160,6 +160,7 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( val heartBeatInfo = thirdPartyAgentHeartbeatUtils.getNewHeartbeat(agentRecord.projectId, agentRecord.id) val lastHeartbeatTime = heartBeatInfo?.heartbeatTime val parallelTaskCount = (agentRecord.parallelTaskCount ?: "").toString() + val dockerParallelTaskCount = (agentRecord.dockerParallelTaskCount ?: "").toString() val agentHostInfo = try { if (needHeartbeatInfo) { AgentHostInfo(nCpus = "0", memTotal = "0", diskTotal = "0") @@ -187,6 +188,7 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( agentInstallPath = agentRecord.agentInstallPath ?: "", maxParallelTaskCount = MAX_PARALLEL_TASK_COUNT, parallelTaskCount = parallelTaskCount, + dockerParallelTaskCount = dockerParallelTaskCount, startedUser = agentRecord.startedUser ?: "", agentUrl = agentUrlService.genAgentUrl(agentRecord), agentScript = agentUrlService.genAgentInstallScript(agentRecord), @@ -641,7 +643,8 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( secretKey = SecurityUtil.decrypt(agentRecord.secretKey), createUser = agentRecord.createdUser, createTime = agentRecord.createdTime.timestamp(), - parallelTaskCount = agentRecord.parallelTaskCount + parallelTaskCount = agentRecord.parallelTaskCount, + dockerParallelTaskCount = agentRecord.dockerParallelTaskCount ) ) } @@ -673,7 +676,8 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( secretKey = SecurityUtil.decrypt(agentRecord.secretKey), createUser = agentRecord.createdUser, createTime = agentRecord.createdTime.timestamp(), - parallelTaskCount = agentRecord.parallelTaskCount + parallelTaskCount = agentRecord.parallelTaskCount, + dockerParallelTaskCount = agentRecord.dockerParallelTaskCount ) ) } @@ -884,7 +888,8 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( secretKey = SecurityUtil.decrypt(it.secretKey), createUser = it.createdUser, createTime = it.createdTime.timestamp(), - parallelTaskCount = it.parallelTaskCount + parallelTaskCount = it.parallelTaskCount, + dockerParallelTaskCount = it.dockerParallelTaskCount ) }.plus(sharedThridPartyAgentList) } @@ -1105,7 +1110,8 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( AgentStatus = AgentStatus.DELETE.name, ParallelTaskCount = -1, envs = mapOf(), - props = mapOf() + props = mapOf(), + dockerParallelTaskCount = -1 ) } @@ -1149,7 +1155,8 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( AgentProps( arch = newHeartbeatInfo.props!!.arch, jdkVersion = newHeartbeatInfo.props!!.jdkVersion ?: listOf(), - userProps = oldUserProps + userProps = oldUserProps, + dockerInitFileInfo = newHeartbeatInfo.props?.dockerInitFileInfo ), false ) @@ -1158,6 +1165,10 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( agentChanged = true } } + if (newHeartbeatInfo.dockerParallelTaskCount != null && agentRecord.dockerParallelTaskCount == null) { + agentRecord.dockerParallelTaskCount = newHeartbeatInfo.dockerParallelTaskCount + agentChanged = true + } if (agentChanged) { thirdPartyAgentDao.saveAgent(context, agentRecord) } @@ -1210,7 +1221,8 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( AgentStatus = AgentStatus.DELETE.name, ParallelTaskCount = -1, envs = mapOf(), - props = mapOf() + props = mapOf(), + dockerParallelTaskCount = -1 ) } if (nodeRecord.nodeIp != newHeartbeatInfo.agentIp || @@ -1245,7 +1257,8 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( }, gateway = agentRecord.gateway, fileGateway = agentRecord.fileGateway, - props = oldUserProps + props = oldUserProps, + dockerParallelTaskCount = agentRecord.dockerParallelTaskCount ?: 0 ) } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/UpgradeService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/UpgradeService.kt index f50ccd8d605..ac7cc0deef0 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/UpgradeService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdPartyAgent/UpgradeService.kt @@ -53,7 +53,7 @@ import java.util.concurrent.TimeUnit import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response -@Suppress("ComplexMethod") +@Suppress("ComplexMethod", "LongMethod") @Service class UpgradeService @Autowired constructor( private val dslContext: DSLContext, @@ -141,6 +141,11 @@ class UpgradeService @Autowired constructor( } } + fun getDockerInitFileMd5(): String { + // 目前仅支持linux且amd和arm脚本上没有区别 + return getRedisValueWithCache(agentGrayUtils.getDockerInitFileMd5Key()) + } + fun getGatewayMapping(): Map { val mappingConfig = getRedisValueWithCache("environment.thirdparty.gateway.mapping") return objectMapper.readValue(mappingConfig) @@ -194,16 +199,27 @@ class UpgradeService @Autowired constructor( val (status, props, os) = checkAgent(projectId, agentId, secretKey) if (status != AgentStatus.IMPORT_OK) { logger.warn("The agent($agentId) status($status) is not OK") - return AgentResult(status, UpgradeItem(agent = false, worker = false, jdk = false)) + return AgentResult( + status, UpgradeItem( + agent = false, + worker = false, + jdk = false, + dockerInitFile = false + ) + ) } if (!checkProjectUpgrade(projectId)) { - return AgentResult(AgentStatus.IMPORT_OK, UpgradeItem(agent = false, worker = false, jdk = false)) + return AgentResult( + AgentStatus.IMPORT_OK, + UpgradeItem(agent = false, worker = false, jdk = false, dockerInitFile = false) + ) } val currentWorkerVersion = getWorkerVersion() val currentGoAgentVersion = getAgentVersion() val currentJdkVersion = getJdkVersion(os, props?.arch) + val currentDockerInitFileMd5 = getDockerInitFileMd5() val canUpgrade = agentGrayUtils.getCanUpgradeAgents().contains(HashUtil.decodeIdToLong(agentId)) @@ -244,11 +260,28 @@ class UpgradeService @Autowired constructor( currentJdkVersion.trim() != info.jdkVersion?.get(2)?.trim())) } + val dockerInitFile = when { + info.dockerInitFileInfo == null -> false + // 目前存在非linux系统的不支持，旧数据或agent不使用docker构建机，所以不校验升级 + info.dockerInitFileInfo?.needUpgrade != true -> false + currentDockerInitFileMd5.isBlank() -> { + logger.warn( + "project: $projectId|agent: $agentId|os: $os|arch: ${props?.arch}|current docker init md5 is null" + ) + false + } + + agentGrayUtils.checkLockUpgrade(agentId, AgentUpgradeType.DOCKER_INIT_FILE) -> false + agentGrayUtils.checkForceUpgrade(agentId, AgentUpgradeType.DOCKER_INIT_FILE) -> true + else -> canUpgrade && info.dockerInitFileInfo?.fileMd5 != currentDockerInitFileMd5 + } + return AgentResult( AgentStatus.IMPORT_OK, UpgradeItem( agent = goAgentVersion, worker = workerVersion, - jdk = jdkVersion + jdk = jdkVersion, + dockerInitFile = dockerInitFile ) ) } diff --git a/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/api/ServiceMetricsResource.kt b/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/api/ServiceMetricsResource.kt new file mode 100644 index 00000000000..3b08d9b4556 --- /dev/null +++ b/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/api/ServiceMetricsResource.kt @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.metrics.api + +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_PROJECT_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.annotation.BkField +import com.tencent.devops.metrics.pojo.vo.BaseQueryReqVO +import com.tencent.devops.metrics.pojo.vo.PipelineSumInfoVO +import com.tencent.devops.metrics.pojo.vo.ThirdPlatformOverviewInfoVO +import io.swagger.annotations.Api +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import javax.ws.rs.Consumes +import javax.ws.rs.GET +import javax.ws.rs.HeaderParam +import javax.ws.rs.POST +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.QueryParam +import javax.ws.rs.core.MediaType + +@Api(tags = ["SERVICE_METRICS"], description = "METRICS") +@Path("/service/metrics/") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface ServiceMetricsResource { + + @ApiOperation("查询流水线汇总信息") + @Path("/summary_pipeline") + @POST + fun queryPipelineSumInfo( + @ApiParam("项目ID", required = true) + @HeaderParam(AUTH_HEADER_DEVOPS_PROJECT_ID) + @BkField(required = true) + projectId: String, + @ApiParam("userId", required = true) + @BkField(required = true) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("查询条件", required = false) + baseQueryReq: BaseQueryReqVO? + ): Result + + @ApiOperation("获取第三方汇总信息") + @Path("/summary_third_party") + @GET + fun queryPipelineSummaryInfo( + @ApiParam("项目ID", required = true) + @HeaderParam(AUTH_HEADER_DEVOPS_PROJECT_ID) + @BkField(required = true) + projectId: String, + @ApiParam("userId", required = true) + @HeaderParam(AUTH_HEADER_USER_ID) + @BkField(required = true) + userId: String, + @ApiParam("开始时间", required = false) + @QueryParam("startTime") + startTime: String?, + @ApiParam("结束时间", required = false) + @QueryParam("endTime") + endTime: String? + ): Result +} diff --git a/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/SaveErrorCodeInfoPO.kt b/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/SaveErrorCodeInfoPO.kt index e0572609d41..7c7b545aeda 100644 --- a/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/SaveErrorCodeInfoPO.kt +++ b/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/SaveErrorCodeInfoPO.kt @@ -37,7 +37,7 @@ data class SaveErrorCodeInfoPO( val id: Long, @ApiModelProperty("错误类型") val errorType: Int, - @ApiModelProperty("错误次数") + @ApiModelProperty("错误码") val errorCode: Int, @ApiModelProperty("错误描述") val errorMsg: String? = null, diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewViewUpdate.kt b/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/UpdateErrorCodeInfoPO.kt similarity index 75% rename from src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewViewUpdate.kt rename to src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/UpdateErrorCodeInfoPO.kt index 3359468ec44..ff29ad33c08 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewViewUpdate.kt +++ b/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/UpdateErrorCodeInfoPO.kt @@ -25,20 +25,22 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.process.pojo.classify +package com.tencent.devops.metrics.pojo.po -import com.tencent.devops.process.pojo.classify.enums.Logic import io.swagger.annotations.ApiModel import io.swagger.annotations.ApiModelProperty +import java.time.LocalDateTime -@ApiModel("") -data class PipelineNewViewUpdate( - @ApiModelProperty("视图名称", required = false) - val name: String, - @ApiModelProperty("是否项目", required = false) - val projected: Boolean, - @ApiModelProperty("逻辑符", required = false) - val logic: Logic, - @ApiModelProperty("流水线视图过滤器列表", required = false) - val filters: List +@ApiModel("保存错误码信息") +data class UpdateErrorCodeInfoPO( + @ApiModelProperty("错误类型") + val errorType: Int, + @ApiModelProperty("错误码") + val errorCode: Int, + @ApiModelProperty("错误描述") + val errorMsg: String? = null, + @ApiModelProperty("修改人") + val modifier: String, + @ApiModelProperty("更新时间") + val updateTime: LocalDateTime ) diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/dao/MetricsDataQueryDao.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/dao/MetricsDataQueryDao.kt index b1e6e36533f..90136c1dff8 100644 --- a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/dao/MetricsDataQueryDao.kt +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/dao/MetricsDataQueryDao.kt @@ -29,13 +29,11 @@ package com.tencent.devops.metrics.dao import com.tencent.devops.model.metrics.tables.TAtomFailSummaryData import com.tencent.devops.model.metrics.tables.TAtomOverviewData -import com.tencent.devops.model.metrics.tables.TErrorCodeInfo import com.tencent.devops.model.metrics.tables.TPipelineFailSummaryData import com.tencent.devops.model.metrics.tables.TPipelineOverviewData import com.tencent.devops.model.metrics.tables.TPipelineStageOverviewData import com.tencent.devops.model.metrics.tables.records.TAtomFailSummaryDataRecord import com.tencent.devops.model.metrics.tables.records.TAtomOverviewDataRecord -import com.tencent.devops.model.metrics.tables.records.TErrorCodeInfoRecord import com.tencent.devops.model.metrics.tables.records.TPipelineFailSummaryDataRecord import com.tencent.devops.model.metrics.tables.records.TPipelineOverviewDataRecord import com.tencent.devops.model.metrics.tables.records.TPipelineStageOverviewDataRecord @@ -142,15 +140,4 @@ class MetricsDataQueryDao { .fetchOne() } } - - fun getErrorCodes( - dslContext: DSLContext, - errorCodes: List - ): Result? { - with(TErrorCodeInfo.T_ERROR_CODE_INFO) { - return dslContext.selectFrom(this) - .where(ERROR_CODE.`in`(errorCodes)) - .fetch() - } - } } diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/dao/MetricsDataReportDao.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/dao/MetricsDataReportDao.kt index 112950e3f3f..15c4e3d1296 100644 --- a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/dao/MetricsDataReportDao.kt +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/dao/MetricsDataReportDao.kt @@ -37,6 +37,7 @@ import com.tencent.devops.metrics.pojo.po.SavePipelineOverviewDataPO import com.tencent.devops.metrics.pojo.po.SavePipelineStageOverviewDataPO import com.tencent.devops.metrics.pojo.po.UpdateAtomFailSummaryDataPO import com.tencent.devops.metrics.pojo.po.UpdateAtomOverviewDataPO +import com.tencent.devops.metrics.pojo.po.UpdateErrorCodeInfoPO import com.tencent.devops.metrics.pojo.po.UpdatePipelineFailSummaryDataPO import com.tencent.devops.metrics.pojo.po.UpdatePipelineOverviewDataPO import com.tencent.devops.metrics.pojo.po.UpdatePipelineStageOverviewDataPO @@ -375,4 +376,39 @@ class MetricsDataReportDao { } } } + + fun saveErrorCodeInfo( + dslContext: DSLContext, + saveErrorCodeInfoPO: SaveErrorCodeInfoPO + ) { + with(TErrorCodeInfo.T_ERROR_CODE_INFO) { + dslContext.insertInto(this) + .set(ID, saveErrorCodeInfoPO.id) + .set(ERROR_TYPE, saveErrorCodeInfoPO.errorType) + .set(ERROR_CODE, saveErrorCodeInfoPO.errorCode) + .set(ERROR_MSG, saveErrorCodeInfoPO.errorMsg) + .set(CREATOR, saveErrorCodeInfoPO.creator) + .set(MODIFIER, saveErrorCodeInfoPO.modifier) + .set(UPDATE_TIME, saveErrorCodeInfoPO.updateTime) + .set(CREATE_TIME, saveErrorCodeInfoPO.createTime) + .execute() + } + } + + fun updateErrorCodeInfo( + dslContext: DSLContext, + updateErrorCodeInfoPO: UpdateErrorCodeInfoPO + ) { + with(TErrorCodeInfo.T_ERROR_CODE_INFO) { + dslContext.update(this) + .set(ERROR_MSG, updateErrorCodeInfoPO.errorMsg) + .set(MODIFIER, updateErrorCodeInfoPO.modifier) + .set(UPDATE_TIME, updateErrorCodeInfoPO.updateTime) + .where( + ERROR_CODE.eq(updateErrorCodeInfoPO.errorCode) + .and(ERROR_TYPE.eq(updateErrorCodeInfoPO.errorType)) + ) + .execute() + } + } } diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/resources/ServiceMetricsResourceImpl.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/resources/ServiceMetricsResourceImpl.kt new file mode 100644 index 00000000000..a31f840d4b0 --- /dev/null +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/resources/ServiceMetricsResourceImpl.kt @@ -0,0 +1,94 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.metrics.resources + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.metrics.api.ServiceMetricsResource +import com.tencent.devops.metrics.pojo.dto.QueryPipelineOverviewDTO +import com.tencent.devops.metrics.pojo.dto.QueryPipelineSummaryInfoDTO +import com.tencent.devops.metrics.pojo.vo.BaseQueryReqVO +import com.tencent.devops.metrics.pojo.vo.PipelineSumInfoVO +import com.tencent.devops.metrics.pojo.vo.ThirdPlatformOverviewInfoVO +import com.tencent.devops.metrics.service.PipelineOverviewManageService +import com.tencent.devops.metrics.service.ThirdPartyManageService +import com.tencent.devops.metrics.utils.QueryParamCheckUtil + +@RestResource +class ServiceMetricsResourceImpl constructor( + private val thirdPartyManageService: ThirdPartyManageService, + private val pipelineOverviewManageService: PipelineOverviewManageService +) : ServiceMetricsResource { + + override fun queryPipelineSumInfo( + projectId: String, + userId: String, + baseQueryReq: BaseQueryReqVO? + ): Result { + val queryReq = baseQueryReq ?: BaseQueryReqVO() + if (queryReq.startTime.isNullOrBlank()) { + queryReq.startTime = QueryParamCheckUtil.getStartDateTime() + } + if (queryReq.endTime.isNullOrBlank()) { + queryReq.endTime = QueryParamCheckUtil.getEndDateTime() + } + QueryParamCheckUtil.checkDateInterval(queryReq.startTime!!, queryReq.endTime!!) + return Result( + PipelineSumInfoVO( + pipelineOverviewManageService.queryPipelineSumInfo( + QueryPipelineOverviewDTO( + projectId = projectId, + userId = userId, + baseQueryReq = queryReq + ) + ) + ) + ) + } + + override fun queryPipelineSummaryInfo( + projectId: String, + userId: String, + startTime: String?, + endTime: String? + ): Result { + val startDateTime = if (!startTime.isNullOrBlank()) startTime else QueryParamCheckUtil.getStartDateTime() + val endDateTime = if (!endTime.isNullOrBlank()) endTime else QueryParamCheckUtil.getEndDateTime() + QueryParamCheckUtil.checkDateInterval(startDateTime, endDateTime) + return Result( + thirdPartyManageService.queryPipelineSummaryInfo( + QueryPipelineSummaryInfoDTO( + projectId = projectId, + userId = userId, + startTime = startDateTime, + endTime = endDateTime + ) + ) + ) + } +} diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/MetricsDataReportServiceImpl.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/MetricsDataReportServiceImpl.kt index f9b1d8ac73c..98b0c5f6d04 100644 --- a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/MetricsDataReportServiceImpl.kt +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/MetricsDataReportServiceImpl.kt @@ -46,10 +46,12 @@ import com.tencent.devops.metrics.pojo.po.SavePipelineOverviewDataPO import com.tencent.devops.metrics.pojo.po.SavePipelineStageOverviewDataPO import com.tencent.devops.metrics.pojo.po.UpdateAtomFailSummaryDataPO import com.tencent.devops.metrics.pojo.po.UpdateAtomOverviewDataPO +import com.tencent.devops.metrics.pojo.po.UpdateErrorCodeInfoPO import com.tencent.devops.metrics.pojo.po.UpdatePipelineFailSummaryDataPO import com.tencent.devops.metrics.pojo.po.UpdatePipelineOverviewDataPO import com.tencent.devops.metrics.pojo.po.UpdatePipelineStageOverviewDataPO import com.tencent.devops.metrics.service.MetricsDataReportService +import com.tencent.devops.metrics.utils.ErrorCodeInfoCacheUtil import com.tencent.devops.model.metrics.tables.records.TAtomOverviewDataRecord import com.tencent.devops.project.api.service.ServiceAllocIdResource import org.jooq.DSLContext @@ -57,6 +59,7 @@ import org.jooq.Result import org.jooq.impl.DSL import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired +import org.springframework.dao.DuplicateKeyException import org.springframework.stereotype.Service import java.math.BigDecimal import java.time.LocalDateTime @@ -176,7 +179,23 @@ class MetricsDataReportServiceImpl @Autowired constructor( metricsDataReportDao.batchSaveAtomFailDetailData(dslContext, saveAtomFailDetailDataPOs) } if (saveErrorCodeInfoPOs.isNotEmpty()) { - metricsDataReportDao.batchSaveErrorCodeInfo(context, saveErrorCodeInfoPOs) + saveErrorCodeInfoPOs.forEach { saveErrorCodeInfoPO -> + try { + metricsDataReportDao.saveErrorCodeInfo(dslContext, saveErrorCodeInfoPO) + } catch (ignored: DuplicateKeyException) { + logger.warn("fail to update errorCodeInfo:$saveErrorCodeInfoPO", ignored) + metricsDataReportDao.updateErrorCodeInfo( + dslContext = dslContext, + updateErrorCodeInfoPO = UpdateErrorCodeInfoPO( + errorType = saveErrorCodeInfoPO.errorType, + errorCode = saveErrorCodeInfoPO.errorCode, + errorMsg = saveErrorCodeInfoPO.errorMsg, + modifier = saveErrorCodeInfoPO.modifier, + updateTime = LocalDateTime.now() + ) + ) + } + } } } logger.info("[$projectId|$pipelineId|$buildId]|end metricsDataReport") @@ -311,6 +330,7 @@ class MetricsDataReportServiceImpl @Autowired constructor( if (taskErrorCode != null) { addErrorCodeInfo( saveErrorCodeInfoPOs = saveErrorCodeInfoPOs, + atomCode = taskMetricsData.atomCode, errorType = taskErrorType, errorCode = taskErrorCode, errorMsg = taskMetricsData.errorMsg, @@ -625,6 +645,7 @@ class MetricsDataReportServiceImpl @Autowired constructor( // 添加错误信息 addErrorCodeInfo( saveErrorCodeInfoPOs = saveErrorCodeInfoPOs, + atomCode = errorInfo.atomCode, errorType = errorType, errorCode = errorCode, errorMsg = errorMsg, @@ -762,24 +783,33 @@ class MetricsDataReportServiceImpl @Autowired constructor( private fun addErrorCodeInfo( saveErrorCodeInfoPOs: MutableSet, + atomCode: String, errorType: Int, errorCode: Int, errorMsg: String?, startUser: String, currentTime: LocalDateTime ) { - saveErrorCodeInfoPOs.add( - SaveErrorCodeInfoPO( - id = client.get(ServiceAllocIdResource::class) - .generateSegmentId("METRICS_ERROR_CODE_INFO").data ?: 0, - errorType = errorType, - errorCode = errorCode, - errorMsg = errorMsg, - creator = startUser, - modifier = startUser, - createTime = currentTime, - updateTime = currentTime + // 从本地缓存获取错误码信息 + val cacheKey = "$atomCode:$errorType:$errorCode" + val errorCodeInfo = ErrorCodeInfoCacheUtil.getIfPresent(cacheKey) + if (errorCodeInfo != null) { + // 缓存中不存在则需要入库 + saveErrorCodeInfoPOs.add( + SaveErrorCodeInfoPO( + id = client.get(ServiceAllocIdResource::class) + .generateSegmentId("METRICS_ERROR_CODE_INFO").data ?: 0, + errorType = errorType, + errorCode = errorCode, + errorMsg = errorMsg, + creator = startUser, + modifier = startUser, + createTime = currentTime, + updateTime = currentTime + ) ) - ) + // 将错误码信息放入缓存中 + ErrorCodeInfoCacheUtil.put(cacheKey, true) + } } } diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/ThirdPartyServiceImpl.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/ThirdPartyServiceImpl.kt index 3fb88ccec4a..314eea4cf80 100644 --- a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/ThirdPartyServiceImpl.kt +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/ThirdPartyServiceImpl.kt @@ -78,7 +78,7 @@ class ThirdPartyServiceImpl @Autowired constructor( val qualityInterceptionRate = if (executeNum == null || interceptionCount == null || executeNum == 0) null else { - if (executeNum == interceptionCount) 0.0 + if (executeNum == interceptionCount) 100.0 else String.format("%.2f", interceptionCount.toDouble() * 100 / executeNum.toDouble()).toDouble() } return ThirdPlatformOverviewInfoVO( diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/utils/ErrorCodeInfoCacheUtil.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/utils/ErrorCodeInfoCacheUtil.kt new file mode 100644 index 00000000000..c5b85da4639 --- /dev/null +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/utils/ErrorCodeInfoCacheUtil.kt @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.metrics.utils + +import com.github.benmanes.caffeine.cache.Caffeine +import java.util.concurrent.TimeUnit + +/** + * 错误码信息缓存 + * + * @since: 2022-12-09 + * @version: $Revision$ $Date$ $LastChangedBy$ + * + */ +object ErrorCodeInfoCacheUtil { + + private val shardingRoutingCache = Caffeine.newBuilder() + .maximumSize(5000) + .expireAfterWrite(7, TimeUnit.DAYS) + .build() + + /** + * 保存错误码信息缓存 + * @param key 缓存key (atomCode:errorType:errorCode) + * @param value 缓存value + */ + fun put(key: String, value: Boolean) { + shardingRoutingCache.put(key, value) + } + + /** + * 从缓存中获取错误码信息 + * @param key 缓存key + * @return 缓存value + */ + fun getIfPresent(key: String): Boolean? { + return shardingRoutingCache.getIfPresent(key) + } +} diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt index a63f84c3c49..c86aea43e99 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt @@ -224,8 +224,8 @@ class DataSourceConfig { username = datasourceUsername password = datasourcePassword driverClassName = Driver::class.java.name - minimumIdle = 10 - maximumPoolSize = 50 + minimumIdle = 1 + maximumPoolSize = 8 idleTimeout = 60000 connectionInitSql = datasourceInitSql leakDetectionThreshold = datasouceLeakDetectionThreshold diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/process/ProcessDataClearDao.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/process/ProcessDataClearDao.kt index 5603fc84aeb..d3c544e2df0 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/process/ProcessDataClearDao.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/process/ProcessDataClearDao.kt @@ -45,6 +45,7 @@ import com.tencent.devops.model.process.tables.TPipelineResourceVersion import com.tencent.devops.model.process.tables.TPipelineSetting import com.tencent.devops.model.process.tables.TPipelineSettingVersion import com.tencent.devops.model.process.tables.TPipelineTimer +import com.tencent.devops.model.process.tables.TPipelineViewGroup import com.tencent.devops.model.process.tables.TPipelineWebhook import com.tencent.devops.model.process.tables.TPipelineWebhookBuildParameter import com.tencent.devops.model.process.tables.TReport @@ -268,4 +269,13 @@ class ProcessDataClearDao { .execute() } } + + fun deletePipelineViewGroup(dslContext: DSLContext, projectId: String, pipelineId: String) { + with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + dslContext.deleteFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.eq(pipelineId)) + .execute() + } + } } diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/process/ProcessDataClearService.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/process/ProcessDataClearService.kt index 4e6111a2a3f..cc2b9d1c8dc 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/process/ProcessDataClearService.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/process/ProcessDataClearService.kt @@ -64,6 +64,7 @@ class ProcessDataClearService @Autowired constructor( processDataClearDao.deleteTemplatePipelineByPipelineId(context, projectId, pipelineId) processDataClearDao.deletePipelineBuildSummaryByPipelineId(context, projectId, pipelineId) processDataClearDao.deletePipelineTemplateAcrossInfo(context, projectId, pipelineId) + processDataClearDao.deletePipelineViewGroup(context, projectId, pipelineId) // 添加删除记录，插入要实现幂等 processDao.addPipelineDataClear( dslContext = context, diff --git a/src/backend/ci/core/openapi/api-openapi/build.gradle.kts b/src/backend/ci/core/openapi/api-openapi/build.gradle.kts index d4064247c91..2062a32cc93 100644 --- a/src/backend/ci/core/openapi/api-openapi/build.gradle.kts +++ b/src/backend/ci/core/openapi/api-openapi/build.gradle.kts @@ -36,6 +36,7 @@ dependencies { api(project(":core:auth:api-auth")) api(project(":core:process:api-process")) api(project(":core:project:api-project")) + api(project(":core:metrics:api-metrics")) api("com.tencent.devops:devops-boot-starter-api") api("com.tencent.bk.devops.turbo:api-turbo:0.0.2-RELEASE") { isTransitive = false diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwArtifactoryResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwArtifactoryResourceV3.kt index 36141c67c4e..effcafa3dcd 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwArtifactoryResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwArtifactoryResourceV3.kt @@ -68,7 +68,7 @@ interface ApigwArtifactoryResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("版本仓库类型", required = true) @@ -79,7 +79,10 @@ interface ApigwArtifactoryResourceV3 { path: String ): Result - @ApiOperation("根据元数据获取文件", tags = ["v3_app_artifactory_list", "v3_user_artifactory_list"]) + @ApiOperation( + "根据元数据获取文件(注意: 如果需要构建产物的下载url，请单独调用下载接口，如 v3_app_artifactory_userDownloadUrl)", + tags = ["v3_app_artifactory_list", "v3_user_artifactory_list"] + ) @Path("/") @GET fun search( @@ -92,7 +95,7 @@ interface ApigwArtifactoryResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwAuthGrantResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwAuthGrantResourceV3.kt index 5377dcd063f..9bc8b008842 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwAuthGrantResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwAuthGrantResourceV3.kt @@ -16,7 +16,7 @@ import javax.ws.rs.PathParam import javax.ws.rs.Produces import javax.ws.rs.core.MediaType -@Api(tags = ["OPENAPI_AUTh_V3"], description = "OPENAPI-权限相关") +@Api(tags = ["OPENAPI_AUTH_V3"], description = "OPENAPI-权限相关") @Path("/{apigwType:apigw-user|apigw-app|apigw}/v3/auth") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @@ -36,7 +36,7 @@ interface ApigwAuthGrantResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目Id", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, grantInstance: GrantInstanceDTO diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwAuthValidateResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwAuthValidateResourceV3.kt index fababdf6e2f..031caf7d56b 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwAuthValidateResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwAuthValidateResourceV3.kt @@ -17,7 +17,7 @@ import javax.ws.rs.Produces import javax.ws.rs.QueryParam import javax.ws.rs.core.MediaType -@Api(tags = ["OPENAPI_AUTh_V3"], description = "OPENAPI-权限相关") +@Api(tags = ["OPENAPI_AUTH_V3"], description = "OPENAPI-权限相关") @Path("/{apigwType:apigw-user|apigw-app|apigw}/v3/auth/validate") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwBuildResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwBuildResourceV3.kt index cec3b56a0ad..36464978f67 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwBuildResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwBuildResourceV3.kt @@ -45,6 +45,8 @@ import com.tencent.devops.process.pojo.pipeline.ModelDetail import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam +import io.swagger.annotations.Example +import io.swagger.annotations.ExampleProperty import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.HeaderParam @@ -75,14 +77,26 @@ interface ApigwBuildResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @PathParam("pipelineId") pipelineId: String, - @ApiParam("启动参数：map<变量名(string),变量值(string)>", required = true) - values: Map, + @ApiParam( + "启动参数：map<变量名(string),变量值(string)>", required = false, + examples = Example( + value = [ + ExampleProperty( + mediaType = "当需要指定启动时流水线变量 var1 为 foobar 时", value = "{\"var1\": \"foobar\"}" + ), + ExampleProperty( + mediaType = "若流水线没有设置输入变量，则填空", value = "{}" + ) + ] + ) + ) + values: Map?, @ApiParam("手动指定构建版本参数", required = false) @QueryParam("buildNo") buildNo: Int? = null @@ -101,7 +115,7 @@ interface ApigwBuildResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -125,7 +139,7 @@ interface ApigwBuildResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -158,7 +172,7 @@ interface ApigwBuildResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -182,7 +196,7 @@ interface ApigwBuildResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -215,7 +229,7 @@ interface ApigwBuildResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -236,7 +250,7 @@ interface ApigwBuildResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -260,7 +274,7 @@ interface ApigwBuildResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -279,7 +293,10 @@ interface ApigwBuildResourceV3 { reviewRequest: StageReviewRequest? = null ): Result - @ApiOperation("获取构建中的变量值", tags = ["v3_app_build_variables_value", "v3_user_build_variables_value"]) + @ApiOperation( + "获取构建中的变量值(注意：变量具有时效性，只能获取最近一个月的任务数据)", + tags = ["v3_app_build_variables_value", "v3_user_build_variables_value"] + ) @POST @Path("/{buildId}/variables") fun getVariableValue( @@ -292,7 +309,7 @@ interface ApigwBuildResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -312,7 +329,7 @@ interface ApigwBuildResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -332,7 +349,7 @@ interface ApigwBuildResourceV3 { @HeaderParam(AUTH_HEADER_USER_ID) @BkField(required = true) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @BkField(required = true) @PathParam("projectId") projectId: String, diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwCredentialResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwCredentialResourceV3.kt index 891a5e12fe2..368ec6e61c4 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwCredentialResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwCredentialResourceV3.kt @@ -71,7 +71,7 @@ interface ApigwCredentialResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("凭证类型列表，用逗号分隔", required = true) @@ -98,7 +98,7 @@ interface ApigwCredentialResourceV3 { // @ApiParam(value = "apigw Type", required = true) // @PathParam("apigwType") // apigwType: String?, -// @ApiParam("项目ID", required = true) +// @ApiParam("项目ID(项目英文名)", required = true) // @PathParam("projectId") // projectId: String, // @ApiParam("第几页", required = false, defaultValue = "1") @@ -122,7 +122,7 @@ interface ApigwCredentialResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("凭据", required = true) @@ -142,7 +142,7 @@ interface ApigwCredentialResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("凭据ID", required = true) @@ -163,7 +163,7 @@ interface ApigwCredentialResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("凭据ID", required = true) @@ -186,7 +186,7 @@ interface ApigwCredentialResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("凭据ID", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwLogResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwLogResourceV3.kt index b1f5d55e45c..e3fbb309ccf 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwLogResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwLogResourceV3.kt @@ -48,7 +48,7 @@ import javax.ws.rs.QueryParam import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response -@Api(tags = ["OPENAPI_LOG_V2"], description = "OPENAPI-构建日志资源") +@Api(tags = ["OPENAPI_LOG_V3"], description = "OPENAPI-构建日志资源") @Path("/{apigwType:apigw-user|apigw-app|apigw}/v3/projects/{projectId}/pipelines/{pipelineId}/builds/{buildId}/logs") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @@ -67,7 +67,7 @@ interface ApigwLogResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -103,13 +103,13 @@ interface ApigwLogResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam("流水线ID", required = true) + @ApiParam("流水线ID (p-开头)", required = true) @PathParam("pipelineId") pipelineId: String, - @ApiParam("构建ID", required = true) + @ApiParam("构建ID (b-开头)", required = true) @PathParam("buildId") buildId: String, @ApiParam("是否包含调试日志", required = false) @@ -127,10 +127,10 @@ interface ApigwLogResourceV3 { @ApiParam("结尾行号", required = true) @QueryParam("end") end: Long, - @ApiParam("对应elementId", required = false) + @ApiParam("对应elementId (e-开头)", required = false) @QueryParam("tag") tag: String?, - @ApiParam("对应jobId", required = false) + @ApiParam("对应containerHashId (c-开头)", required = false) @QueryParam("jobId") jobId: String?, @ApiParam("执行次数", required = false) @@ -151,13 +151,13 @@ interface ApigwLogResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam("流水线ID", required = true) + @ApiParam("流水线ID (p-开头)", required = true) @PathParam("pipelineId") pipelineId: String, - @ApiParam("构建ID", required = true) + @ApiParam("构建ID (b-开头)", required = true) @PathParam("buildId") buildId: String, @ApiParam("起始行号", required = true) @@ -166,10 +166,10 @@ interface ApigwLogResourceV3 { @ApiParam("是否包含调试日志", required = false) @QueryParam("debug") debug: Boolean? = false, - @ApiParam("对应elementId", required = false) + @ApiParam("对应elementId (e-开头)", required = false) @QueryParam("tag") tag: String?, - @ApiParam("对应jobId", required = false) + @ApiParam("对应containerHashId (c-开头)", required = false) @QueryParam("jobId") jobId: String?, @ApiParam("执行次数", required = false) @@ -177,7 +177,10 @@ interface ApigwLogResourceV3 { executeCount: Int? ): Result - @ApiOperation("下载日志接口", tags = ["v3_user_log_download", "v3_app_log_download"]) + @ApiOperation( + "下载日志接口(注意: 接口返回application/octet-stream数据，Request Header Accept 类型不一致将导致错误)", + tags = ["v3_user_log_download", "v3_app_log_download"] + ) @GET @Path("/download") @Produces(MediaType.APPLICATION_OCTET_STREAM) @@ -191,19 +194,19 @@ interface ApigwLogResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam("流水线ID", required = true) + @ApiParam("流水线ID (p-开头)", required = true) @PathParam("pipelineId") pipelineId: String, - @ApiParam("构建ID", required = true) + @ApiParam("构建ID (b-开头)", required = true) @PathParam("buildId") buildId: String, - @ApiParam("对应element ID", required = false) + @ApiParam("对应element ID (e-开头)", required = false) @QueryParam("tag") tag: String?, - @ApiParam("对应jobId", required = false) + @ApiParam("对应containerHashId (c-开头)", required = false) @QueryParam("jobId") jobId: String?, @ApiParam("执行次数", required = false) @@ -224,16 +227,16 @@ interface ApigwLogResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam("流水线ID", required = true) + @ApiParam("流水线ID (p-开头)", required = true) @PathParam("pipelineId") pipelineId: String, - @ApiParam("构建ID", required = true) + @ApiParam("构建ID (b-开头)", required = true) @PathParam("buildId") buildId: String, - @ApiParam("对应elementId", required = true) + @ApiParam("对应elementId (e-开头)", required = true) @QueryParam("tag") tag: String, @ApiParam("执行次数", required = false) @@ -248,13 +251,13 @@ interface ApigwLogResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam("流水线ID", required = true) + @ApiParam("流水线ID (p-开头)", required = true) @PathParam("pipelineId") pipelineId: String, - @ApiParam("构建ID", required = true) + @ApiParam("构建ID (b-开头)", required = true) @PathParam("buildId") buildId: String ): Result diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwMarketTemplateResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwMarketTemplateResourceV3.kt index 5cdaf683d5c..ccbe9d75851 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwMarketTemplateResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwMarketTemplateResourceV3.kt @@ -43,7 +43,7 @@ import javax.ws.rs.PathParam import javax.ws.rs.Produces import javax.ws.rs.core.MediaType -@Api(tags = ["OPEN_API_MARKET"], description = "OPEN-API-研发市场资源") +@Api(tags = ["OPEN_API_MARKET_V3"], description = "OPEN-API-研发市场资源") @Path("/{apigwType:apigw-user|apigw-app|apigw}/v3/market") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwPipelineGroupResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwPipelineGroupResourceV3.kt index ccfa128ee22..bd8479bb0cd 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwPipelineGroupResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwPipelineGroupResourceV3.kt @@ -62,7 +62,7 @@ interface ApigwPipelineGroupResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String ): Result> @@ -96,7 +96,7 @@ interface ApigwPipelineGroupResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("分组ID", required = true) @@ -111,7 +111,7 @@ interface ApigwPipelineGroupResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "流水线标签创建请求", required = true) @@ -125,7 +125,7 @@ interface ApigwPipelineGroupResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("标签ID", required = true) @@ -140,7 +140,7 @@ interface ApigwPipelineGroupResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "流水线标签更新请求", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwPipelineResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwPipelineResourceV3.kt index 94e8f1eac27..7c6fea4788e 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwPipelineResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwPipelineResourceV3.kt @@ -45,6 +45,8 @@ import com.tencent.devops.process.pojo.setting.PipelineSetting import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam +import io.swagger.annotations.Example +import io.swagger.annotations.ExampleProperty import javax.validation.Valid import javax.ws.rs.Consumes import javax.ws.rs.DELETE @@ -79,7 +81,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "流水线模型", required = true) @@ -100,13 +102,211 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @PathParam("pipelineId") pipelineId: String, - @ApiParam(value = "流水线模型", required = true) + @ApiParam( + value = "流水线模型", required = true, examples = Example( + value = [ + ExampleProperty( + mediaType = "如果我想更改流水线启动变量param的默认值为value2", + value = """ + { + "name": "更改流水线启动变量默认值", + "stages": [{ + "containers": [{ + "@type": "trigger", + "elements": [{ + "@type": "manualTrigger", + "...": "..." + }], + "params": [{ + "id": "param", + "required": true, + "type": "STRING", + "defaultValue": "value2", + "desc": "", + "readOnly": false + }] + }], + "...": "..." + }, { + "containers": [{}], + "...": "..." + }], + "...": "..." + } + """ + ), + ExampleProperty( + mediaType = "如果我想启用或是更改job互斥组配置", + value = """ + { + "stages": [{ + "containers": [{ + "@type": "trigger", + "...": "..." + }], + "...": "..." + }, { + "containers": [{ + "@type" : "vmBuild", + "name": "想要更改互斥组配置的job", + "elements": [{ + "...": "..." + }], + "...": "...", + "mutexGroup": { + "enable": true, + "mutexGroupName": "huchizu", + "queueEnable": true, + "timeout": 900, + "queue": 5 + } + }], + "...": "..." + }], + "...": "..." + } + """ + ), + ExampleProperty( + mediaType = "一般先通过接口(比如v3_app_pipeline_get)拿到编排，再根据自己的需求更改后上传更新", + value = """ + { + "name": "一个非常简单的例子", + "desc": "", + "stages": [{ + "containers": [{ + "@type": "trigger", + "id": "0", + "name": "构建触发", + "elements": [{ + "@type": "manualTrigger", + "name": "手动触发", + "id": "T-1-1-1", + "canElementSkip": true, + "useLatestParameters": false, + "executeCount": 1, + "version": "1.*", + "classType": "manualTrigger", + "elementEnable": true, + "atomCode": "manualTrigger", + "taskAtom": "" + }], + "params": [], + "containerId": "0", + "containerHashId": "c-ccef587f17cd421a8a4e6aadc02777c6", + "executeCount": 0, + "matrixGroupFlag": false, + "classType": "trigger" + }], + "id": "stage-1", + "name": "stage-1", + "tag": ["28ee946a59f64949a74f3dee40a1bda4"], + "fastKill": false, + "finally": false + }, { + "containers": [{ + "@type": "vmBuild", + "id": "1", + "name": "构建环境-LINUX", + "elements": [{ + "@type": "linuxScript", + "name": "Bash", + "id": "e-efc1874f0cae44a0b56eba8b113b83f8", + "scriptType": "SHELL", + "script": "echo \"我只是为了测试\"", + "continueNoneZero": false, + "enableArchiveFile": false, + "archiveFile": "", + "additionalOptions": { + "enable": true, + "continueWhenFailed": false, + "manualSkip": false, + "retryWhenFailed": false, + "retryCount": 1, + "manualRetry": false, + "timeout": 900, + "runCondition": "PRE_TASK_SUCCESS", + "pauseBeforeExec": false, + "subscriptionPauseUser": "devops", + "otherTask": "", + "customVariables": [{ + "key": "param1", + "value": "" + }], + "customCondition": "", + "enableCustomEnv": false, + "customEnv": [{ + "key": "param1", + "value": "" + }] + }, + "executeCount": 1, + "version": "1.*", + "classType": "linuxScript", + "elementEnable": true, + "atomCode": "linuxScript", + "taskAtom": "" + }], + "baseOS": "LINUX", + "vmNames": [], + "maxQueueMinutes": 60, + "maxRunningMinutes": 900, + "buildEnv": {}, + "dispatchType": { + "buildType": "PUBLIC_DEVCLOUD", + "value": "tlinux3_ci", + "imageType": "BKSTORE", + "credentialId": "", + "credentialProject": "", + "imageCode": "tlinux3_ci", + "imageVersion": "2.*", + "imageName": "tlinux3-CI镜像", + "dockerBuildVersion": "tlinux3_ci", + "imagePublicFlag": false, + "imageRDType": "", + "recommendFlag": true + }, + "showBuildResource": false, + "enableExternal": false, + "containerId": "1", + "containerHashId": "c-758f14c0c5e644e1b70f1bf37a1cb5a5", + "executeCount": 0, + "jobId": "job_biS", + "matrixGroupFlag": false, + "nfsSwitch": false, + "classType": "vmBuild" + }], + "id": "stage-2", + "name": "stage-2", + "tag": ["28ee946a59f64949a74f3dee40a1bda4"], + "fastKill": false, + "finally": false, + "checkIn": { + "manualTrigger": false, + "timeout": 24 + }, + "checkOut": { + "manualTrigger": false, + "timeout": 24 + } + }], + "labels": [], + "instanceFromTemplate": false, + "pipelineCreator": "devops", + "events": {}, + "latestVersion": 6 + } + """ + ) + ] + ) + ) pipeline: Model ): Result @@ -123,7 +323,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "流水线模型与设置", required = true) @@ -144,7 +344,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -168,7 +368,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -189,7 +389,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID列表", required = true) @@ -203,7 +403,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "流水线模型", required = true) @@ -226,7 +426,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -247,7 +447,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("第几页", required = false, defaultValue = "1") @@ -271,7 +471,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -292,7 +492,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -315,7 +515,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -336,7 +536,7 @@ interface ApigwPipelineResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwProjectResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwProjectResourceV3.kt index 52b0ae5c42d..be04e9bff9d 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwProjectResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwProjectResourceV3.kt @@ -90,7 +90,7 @@ interface ApigwProjectResourceV3 { @ApiParam("userId", required = true) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "项目信息", required = true) @@ -158,7 +158,7 @@ interface ApigwProjectResourceV3 { @ApiParam("项目名称或者项目英文名") @QueryParam("name") name: String, - @ApiParam("项目ID") + @ApiParam("项目ID(项目英文名)") @QueryParam("english_name") projectId: String? ): Result diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwQualityResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwQualityResourceV3.kt index 74420433c06..e424fbfa9b0 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwQualityResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwQualityResourceV3.kt @@ -70,7 +70,7 @@ interface ApigwQualityResourceV3 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -94,7 +94,7 @@ interface ApigwQualityResourceV3 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -114,7 +114,7 @@ interface ApigwQualityResourceV3 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -137,7 +137,7 @@ interface ApigwQualityResourceV3 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -158,7 +158,7 @@ interface ApigwQualityResourceV3 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -170,7 +170,7 @@ interface ApigwQualityResourceV3 { @ApiParam("规则ID", required = false) @QueryParam("ruleHashId") ruleHashId: String?, - @ApiParam("状态", required = false) + @ApiParam("状态", required = false, type = "ENUM(PASS, FAIL)") @QueryParam("interceptResult") interceptResult: RuleInterceptResult?, @ApiParam("开始时间", required = false) @@ -200,7 +200,7 @@ interface ApigwQualityResourceV3 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwRepositoryCommitResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwRepositoryCommitResourceV3.kt index dba94903577..379ee202749 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwRepositoryCommitResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwRepositoryCommitResourceV3.kt @@ -65,7 +65,7 @@ interface ApigwRepositoryCommitResourceV3 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwRepositoryResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwRepositoryResourceV3.kt index 6cd7ebd316d..d8cb7da62f1 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwRepositoryResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwRepositoryResourceV3.kt @@ -39,6 +39,8 @@ import com.tencent.devops.repository.pojo.RepositoryInfo import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam +import io.swagger.annotations.Example +import io.swagger.annotations.ExampleProperty import javax.ws.rs.Consumes import javax.ws.rs.DELETE import javax.ws.rs.GET @@ -71,7 +73,7 @@ interface ApigwRepositoryResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("仓库类型", required = false) @@ -92,10 +94,29 @@ interface ApigwRepositoryResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam(value = "代码库模型", required = true) + @ApiParam( + value = "代码库模型", required = true, examples = Example( + value = [ + ExampleProperty( + mediaType = "user00通过OAUTH认证给项目关联 Tencent/bk-ci 的github代码库", + value = """ + { + "@type": "github", + "aliasName": "Tencent/bk-ci", + "credentialId": "", + "projectName": "Tencent/bk-ci", + "url": "https://github.com/Tencent/bk-ci.git", + "authType": "OAUTH", + "userName": "user00" + } + """ + ) + ] + ) + ) repository: Repository ): Result @@ -112,7 +133,7 @@ interface ApigwRepositoryResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("代码库哈希ID", required = true) @@ -133,13 +154,78 @@ interface ApigwRepositoryResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("代码库哈希ID", required = true) @PathParam("repositoryHashId") repositoryHashId: String, - @ApiParam(value = "代码库模型", required = true) + @ApiParam( + value = "代码库模型", required = true, examples = Example( + value = [ + ExampleProperty( + mediaType = "如果我想通过oauth关联codeGit类型的代码库", + value = """ + { + "@type": "codeGit", + "aliasName": "devops/test", + "credentialId": "", + "projectName": "devops/test", + "url": "https://www.xxx.com/devops/test.git", + "authType": "OAUTH", + "svnType": "ssh", + "userName": "devops" + } + """ + ), + ExampleProperty( + mediaType = "如果我想关联TGIT类型的代码库，只能通过HTTP，需要使用凭据test", + value = """ + { + "@type": "codeTGit", + "aliasName": "devops/test", + "credentialId": "test", + "projectName": "devops/test", + "url": "https://git.tencent.com/devops/test.git", + "authType": "HTTPS", + "svnType": "ssh", + "userName": "devops" + } + """ + ), + ExampleProperty( + mediaType = "如果我想关联GitHub类型的代码库，只能通过Oauth", + value = """ + { + "@type": "github", + "aliasName": "Tencent/bk-ci", + "credentialId": "", + "projectName": "Tencent/bk-ci", + "url": "https://github.com/Tencent/bk-ci.git", + "authType": "OAUTH", + "svnType": "ssh", + "userName": "devops" + } + """ + ), + ExampleProperty( + mediaType = "如果我想关联P4类型的代码库，只能通过HTTP，需要使用凭据test", + value = """ + { + "@type": "codeP4", + "aliasName": "devops/test", + "credentialId": "test", + "projectName": "localhost:1666", + "url": "localhost:1666", + "authType": "HTTP", + "svnType": "ssh", + "userName": "devops" + } + """ + ) + ] + ) + ) repository: Repository ): Result } diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwTemplateInstanceResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwTemplateInstanceResourceV3.kt index c7bff4d21e1..51ac16a9f2b 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwTemplateInstanceResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwTemplateInstanceResourceV3.kt @@ -43,6 +43,8 @@ import com.tencent.devops.process.pojo.template.TemplateOperationRet import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam +import io.swagger.annotations.Example +import io.swagger.annotations.ExampleProperty import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.HeaderParam @@ -73,19 +75,59 @@ interface ApigwTemplateInstanceResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) @PathParam("templateId") templateId: String, - @ApiParam("模板版本", required = true) + @ApiParam("模板版本（可通过v3_app_template_list接口获取）", required = true) @QueryParam("version") version: Long, @ApiParam("是否应用模板设置") @QueryParam("useTemplateSettings") useTemplateSettings: Boolean, - @ApiParam("创建实例", required = true) + @ApiParam( + "创建实例", required = true, examples = Example( + value = [ + ExampleProperty( + mediaType = "如果我想简单的实例化两条无启动变量的流水线1和2", + value = """ + [ + { + "pipelineName": "1", + "param": [] + }, + { + "pipelineName": "2", + "param": [] + } + ] + """ + ), + ExampleProperty( + mediaType = "如果我想实例化一条带启动变量param1的流水线3", + value = """ + [ + { + "pipelineName": "3", + "param": [ + { + "id": "param1", + "required": true, + "type": "STRING //可以是其他类型，以实际情况为准", + "defaultValue": "param1的值", + "desc": "", + "readOnly": false + } + ] + } + ] + """ + ) + ] + ) + ) instances: List ): TemplateOperationRet @@ -102,13 +144,13 @@ interface ApigwTemplateInstanceResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) @PathParam("templateId") templateId: String, - @ApiParam("版本名", required = true) + @ApiParam("版本名（可通过v3_app_template_list接口获取）", required = true) @QueryParam("version") version: Long, @ApiParam("是否应用模板设置") @@ -131,7 +173,7 @@ interface ApigwTemplateInstanceResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwTemplateResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwTemplateResourceV3.kt index f234d8ffd32..8f1fa509c2d 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwTemplateResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/ApigwTemplateResourceV3.kt @@ -101,7 +101,7 @@ interface ApigwTemplateResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) @@ -125,7 +125,7 @@ interface ApigwTemplateResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模版类型", required = false) @@ -156,7 +156,7 @@ interface ApigwTemplateResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板", required = true) @@ -176,7 +176,7 @@ interface ApigwTemplateResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) @@ -197,7 +197,7 @@ interface ApigwTemplateResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) @@ -221,7 +221,7 @@ interface ApigwTemplateResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/environment/ApigwEnvironmentResourceV3.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/environment/ApigwEnvironmentResourceV3.kt index bf4582d8579..d74ebc02c50 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/environment/ApigwEnvironmentResourceV3.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v3/environment/ApigwEnvironmentResourceV3.kt @@ -73,7 +73,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String ): Result> @@ -91,7 +91,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "环境信息", required = true) @@ -111,7 +111,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境 hashId", required = true) @@ -132,7 +132,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "节点列表", required = true) @@ -152,7 +152,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境 hashId", required = true) @@ -175,7 +175,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境 hashId", required = true) @@ -198,7 +198,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String ): Result> @@ -219,7 +219,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境名称(s)", required = true) @@ -242,7 +242,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境 hashId(s)", required = true) @@ -265,7 +265,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("节点 hashIds", required = true) @@ -288,7 +288,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("节点 hashIds", required = true) @@ -308,7 +308,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String ): Result> @@ -329,7 +329,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("节点 hashId", required = true) @@ -350,7 +350,7 @@ interface ApigwEnvironmentResourceV3 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境 hashId", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwArtifactoryResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwArtifactoryResourceV4.kt index c151167974e..015bc6705af 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwArtifactoryResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwArtifactoryResourceV4.kt @@ -71,7 +71,7 @@ interface ApigwArtifactoryResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("版本仓库类型", required = true) @@ -109,7 +109,10 @@ interface ApigwArtifactoryResourceV4 { path: String ): Result - @ApiOperation("根据元数据获取文件", tags = ["v4_app_artifactory_list", "v4_user_artifactory_list"]) + @ApiOperation( + "根据元数据获取文件(注意: 如果需要构建产物的下载url，请单独调用下载接口，如 v4_app_artifactory_userDownloadUrl)", + tags = ["v4_app_artifactory_list", "v4_user_artifactory_list"] + ) @Path("/file_info") @GET fun search( @@ -122,7 +125,7 @@ interface ApigwArtifactoryResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = false) @@ -179,7 +182,7 @@ interface ApigwArtifactoryResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("文件路径", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthGrantResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthGrantResourceV4.kt index 4558528ee95..ed1378e66db 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthGrantResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthGrantResourceV4.kt @@ -16,7 +16,7 @@ import javax.ws.rs.PathParam import javax.ws.rs.Produces import javax.ws.rs.core.MediaType -@Api(tags = ["OPENAPI_AUTh_V4"], description = "OPENAPI-权限相关") +@Api(tags = ["OPENAPI_AUTH_V4"], description = "OPENAPI-权限相关") @Path("/{apigwType:apigw-user|apigw-app|apigw}/v4/auth/projects/{projectId}") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @@ -36,7 +36,7 @@ interface ApigwAuthGrantResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目Id", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, grantInstance: GrantInstanceDTO diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthValidateResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthValidateResourceV4.kt index 25cc49631d2..806c10b2345 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthValidateResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthValidateResourceV4.kt @@ -17,7 +17,7 @@ import javax.ws.rs.Produces import javax.ws.rs.QueryParam import javax.ws.rs.core.MediaType -@Api(tags = ["OPENAPI_AUTh_V4"], description = "OPENAPI-权限相关") +@Api(tags = ["OPENAPI_AUTH_V4"], description = "OPENAPI-权限相关") @Path("/{apigwType:apigw-user|apigw-app|apigw}/v4/auth/validate/projects/{projectId}") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwBuildResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwBuildResourceV4.kt index 388dfbc706c..9ebf44d37c8 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwBuildResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwBuildResourceV4.kt @@ -51,6 +51,8 @@ import com.tencent.devops.process.pojo.pipeline.ModelDetail import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam +import io.swagger.annotations.Example +import io.swagger.annotations.ExampleProperty import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.HeaderParam @@ -81,14 +83,26 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @QueryParam("pipelineId") pipelineId: String, - @ApiParam("启动参数：map<变量名(string),变量值(string)>", required = true) - values: Map, + @ApiParam( + "启动参数：map<变量名(string),变量值(string)>", required = false, + examples = Example( + value = [ + ExampleProperty( + mediaType = "当需要指定启动时流水线变量 var1 为 foobar 时", value = "{\"var1\": \"foobar\"}" + ), + ExampleProperty( + mediaType = "若流水线没有设置输入变量，则填空", value = "{}" + ) + ] + ) + ) + values: Map?, @ApiParam("手动指定构建版本参数", required = false) @QueryParam("buildNo") buildNo: Int? = null @@ -107,7 +121,7 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -131,7 +145,7 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -164,7 +178,7 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = false) @@ -188,7 +202,7 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -227,22 +241,22 @@ interface ApigwBuildResourceV4 { @ApiParam("触发方式", required = false) @QueryParam("trigger") trigger: List?, - @ApiParam("排队于-开始时间(时间戳形式)", required = false) + @ApiParam("排队于-开始时间(时间戳毫秒级别，13位数字)", required = false) @QueryParam("queueTimeStartTime") queueTimeStartTime: Long?, - @ApiParam("排队于-结束时间(时间戳形式)", required = false) + @ApiParam("排队于-结束时间(时间戳毫秒级别，13位数字)", required = false) @QueryParam("queueTimeEndTime") queueTimeEndTime: Long?, - @ApiParam("开始于-开始时间(时间戳形式)", required = false) + @ApiParam("开始于-流水线的执行开始时间(时间戳毫秒级别，13位数字)", required = false) @QueryParam("startTimeStartTime") startTimeStartTime: Long?, - @ApiParam("开始于-结束时间(时间戳形式)", required = false) + @ApiParam("开始于-流水线的执行结束时间(时间戳毫秒级别，13位数字)", required = false) @QueryParam("startTimeEndTime") startTimeEndTime: Long?, - @ApiParam("结束于-开始时间(时间戳形式)", required = false) + @ApiParam("结束于-流水线的执行开始时间(时间戳毫秒级别，13位数字)", required = false) @QueryParam("endTimeStartTime") endTimeStartTime: Long?, - @ApiParam("结束于-结束时间(时间戳形式)", required = false) + @ApiParam("结束于-流水线的执行结束时间(时间戳毫秒级别，13位数字)", required = false) @QueryParam("endTimeEndTime") endTimeEndTime: Long?, @ApiParam("耗时最小值", required = false) @@ -263,7 +277,7 @@ interface ApigwBuildResourceV4 { @ApiParam("构建信息", required = false) @QueryParam("buildMsg") buildMsg: String?, - @ApiParam("触发人", required = false) + @ApiParam("执行人", required = false) @QueryParam("startUser") startUser: List? ): Result> @@ -281,7 +295,7 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -302,7 +316,7 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -326,7 +340,7 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = false) @@ -345,7 +359,10 @@ interface ApigwBuildResourceV4 { reviewRequest: StageReviewRequest? = null ): Result - @ApiOperation("获取构建中的变量值", tags = ["v4_app_build_variables_value", "v4_user_build_variables_value"]) + @ApiOperation( + "获取构建中的变量值(注意：变量具有时效性，只能获取最近一个月的任务数据)", + tags = ["v4_app_build_variables_value", "v4_user_build_variables_value"] + ) @POST @Path("/build_variables") fun getVariableValue( @@ -358,7 +375,7 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = false) @@ -378,7 +395,7 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = false) @@ -398,7 +415,7 @@ interface ApigwBuildResourceV4 { @HeaderParam(AUTH_HEADER_USER_ID) @BkField(required = true) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @BkField(required = true) @PathParam("projectId") projectId: String, @@ -443,7 +460,7 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = false) @@ -472,7 +489,7 @@ interface ApigwBuildResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwCredentialResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwCredentialResourceV4.kt index 5ff84871f3d..4adeca849d9 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwCredentialResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwCredentialResourceV4.kt @@ -71,7 +71,7 @@ interface ApigwCredentialResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("凭证类型列表，用逗号分隔", required = true) @@ -101,7 +101,7 @@ interface ApigwCredentialResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("凭据", required = true) @@ -121,7 +121,7 @@ interface ApigwCredentialResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("凭据ID", required = true) @@ -142,7 +142,7 @@ interface ApigwCredentialResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("凭据ID", required = true) @@ -165,7 +165,7 @@ interface ApigwCredentialResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("凭据ID", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwLogResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwLogResourceV4.kt index f1a0fb626d1..e05fe018cf4 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwLogResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwLogResourceV4.kt @@ -67,22 +67,22 @@ interface ApigwLogResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam("流水线ID", required = false) + @ApiParam("流水线ID (p-开头)", required = false) @QueryParam("pipelineId") pipelineId: String?, - @ApiParam("构建ID", required = true) + @ApiParam("构建ID (b-开头)", required = true) @QueryParam("buildId") buildId: String, @ApiParam("是否包含调试日志", required = false) @QueryParam("debug") debug: Boolean? = false, - @ApiParam("对应elementId", required = false) + @ApiParam("对应elementId (e-开头)", required = false) @QueryParam("tag") elementId: String?, - @ApiParam("对应jobId", required = false) + @ApiParam("对应containerHashId (c-开头)", required = false) @QueryParam("jobId") jobId: String?, @ApiParam("执行次数", required = false) @@ -103,13 +103,13 @@ interface ApigwLogResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam("流水线ID", required = false) + @ApiParam("流水线ID (p-开头)", required = false) @QueryParam("pipelineId") pipelineId: String?, - @ApiParam("构建ID", required = true) + @ApiParam("构建ID (b-开头)", required = true) @QueryParam("buildId") buildId: String, @ApiParam("是否包含调试日志", required = false) @@ -127,10 +127,10 @@ interface ApigwLogResourceV4 { @ApiParam("结尾行号", required = true) @QueryParam("end") end: Long, - @ApiParam("对应elementId", required = false) + @ApiParam("对应elementId (e-开头)", required = false) @QueryParam("tag") tag: String?, - @ApiParam("对应jobId", required = false) + @ApiParam("对应containerHashId (c-开头)", required = false) @QueryParam("jobId") jobId: String?, @ApiParam("执行次数", required = false) @@ -151,13 +151,13 @@ interface ApigwLogResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam("流水线ID", required = false) + @ApiParam("流水线ID (p-开头)", required = false) @QueryParam("pipelineId") pipelineId: String?, - @ApiParam("构建ID", required = true) + @ApiParam("构建ID (b-开头)", required = true) @QueryParam("buildId") buildId: String, @ApiParam("起始行号", required = true) @@ -166,10 +166,10 @@ interface ApigwLogResourceV4 { @ApiParam("是否包含调试日志", required = false) @QueryParam("debug") debug: Boolean? = false, - @ApiParam("对应elementId", required = false) + @ApiParam("对应elementId (e-开头)", required = false) @QueryParam("tag") tag: String?, - @ApiParam("对应jobId", required = false) + @ApiParam("对应containerHashId (c-开头)", required = false) @QueryParam("jobId") jobId: String?, @ApiParam("执行次数", required = false) @@ -177,7 +177,10 @@ interface ApigwLogResourceV4 { executeCount: Int? ): Result - @ApiOperation("下载日志接口", tags = ["v4_user_log_download", "v4_app_log_download"]) + @ApiOperation( + "下载日志接口(注意: 接口返回application/octet-stream数据，Request Header Accept 类型不一致将导致错误)", + tags = ["v4_user_log_download", "v4_app_log_download"] + ) @GET @Path("/download_logs") @Produces(MediaType.APPLICATION_OCTET_STREAM) @@ -191,19 +194,19 @@ interface ApigwLogResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam("流水线ID", required = false) + @ApiParam("流水线ID (p-开头)", required = false) @QueryParam("pipelineId") pipelineId: String?, - @ApiParam("构建ID", required = true) + @ApiParam("构建ID (b-开头)", required = true) @QueryParam("buildId") buildId: String, - @ApiParam("对应element ID", required = false) + @ApiParam("对应element ID (e-开头)", required = false) @QueryParam("tag") tag: String?, - @ApiParam("对应jobId", required = false) + @ApiParam("对应containerHashId (c-开头)", required = false) @QueryParam("jobId") jobId: String?, @ApiParam("执行次数", required = false) @@ -224,16 +227,16 @@ interface ApigwLogResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam("流水线ID", required = false) + @ApiParam("流水线ID (p-开头)", required = false) @QueryParam("pipelineId") pipelineId: String?, - @ApiParam("构建ID", required = true) + @ApiParam("构建ID (b-开头)", required = true) @QueryParam("buildId") buildId: String, - @ApiParam("对应elementId", required = true) + @ApiParam("对应elementId (e-开头)", required = true) @QueryParam("tag") tag: String, @ApiParam("执行次数", required = false) @@ -248,13 +251,13 @@ interface ApigwLogResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, - @ApiParam("流水线ID", required = true) + @ApiParam("流水线ID (p-开头)", required = true) @QueryParam("pipelineId") pipelineId: String, - @ApiParam("构建ID", required = true) + @ApiParam("构建ID (b-开头)", required = true) @QueryParam("buildId") buildId: String ): Result diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwMarketTemplateResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwMarketTemplateResourceV4.kt index 9ed3466ff9a..136dba0c42f 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwMarketTemplateResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwMarketTemplateResourceV4.kt @@ -43,7 +43,7 @@ import javax.ws.rs.PathParam import javax.ws.rs.Produces import javax.ws.rs.core.MediaType -@Api(tags = ["OPEN_API_MARKET"], description = "OPEN-API-研发市场资源") +@Api(tags = ["OPEN_API_MARKET_V4"], description = "OPEN-API-研发市场资源") @Path("/{apigwType:apigw-user|apigw-app|apigw}/v4/market") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwMetricsResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwMetricsResourceV4.kt new file mode 100644 index 00000000000..4f8d1232b75 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwMetricsResourceV4.kt @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.api.apigw.v4 + +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_USER_ID +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.openapi.pojo.ApigwMetricsSummary +import io.swagger.annotations.Api +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import javax.ws.rs.Consumes +import javax.ws.rs.GET +import javax.ws.rs.HeaderParam +import javax.ws.rs.Path +import javax.ws.rs.PathParam +import javax.ws.rs.Produces +import javax.ws.rs.core.MediaType + +@Api(tags = ["OPENAPI_METRICS_V4"], description = "metrics接口") +@Path("/{apigwType:apigw-user|apigw-app|apigw}/v4/metrics/projectId/{projectId}") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Suppress("ALL") +interface ApigwMetricsResourceV4 { + + @GET + @ApiOperation("获取看板 summary 数据", tags = ["v4_app_metrics_summary", "v4_user_metrics_summary"]) + @Path("/summary") + fun getSummaryInfo( + @ApiParam(value = "项目id", required = true) + @PathParam("projectId") + projectId: String, + @ApiParam(value = "用户信息", required = true) + @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) + userId: String + ): Result +} diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPermissionMoveResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPermissionMoveResourceV4.kt index 66feec637e6..cc723191c1c 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPermissionMoveResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPermissionMoveResourceV4.kt @@ -17,7 +17,7 @@ import javax.ws.rs.Produces import javax.ws.rs.QueryParam import javax.ws.rs.core.MediaType -@Api(tags = ["OPEN_API_MOVE"], description = "OPEN-API-迁移") +@Api(tags = ["OPEN_API_MOVE_V4"], description = "OPEN-API-迁移") @Path("/{apigwType:apigw-user|apigw-app|apigw}/v4/permission/move/projects/{projectId}") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPipelineGroupResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPipelineGroupResourceV4.kt index 7599bdd9fff..aedec105124 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPipelineGroupResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPipelineGroupResourceV4.kt @@ -63,7 +63,7 @@ interface ApigwPipelineGroupResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String ): Result> @@ -97,7 +97,7 @@ interface ApigwPipelineGroupResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("分组ID", required = true) @@ -112,7 +112,7 @@ interface ApigwPipelineGroupResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "流水线标签创建请求", required = true) @@ -126,7 +126,7 @@ interface ApigwPipelineGroupResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("标签ID", required = true) @@ -141,7 +141,7 @@ interface ApigwPipelineGroupResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "流水线标签更新请求", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPipelineResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPipelineResourceV4.kt index d09b77755b4..5521c556ea2 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPipelineResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPipelineResourceV4.kt @@ -46,6 +46,8 @@ import com.tencent.devops.process.pojo.setting.PipelineSetting import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam +import io.swagger.annotations.Example +import io.swagger.annotations.ExampleProperty import javax.validation.Valid import javax.ws.rs.Consumes import javax.ws.rs.DELETE @@ -80,7 +82,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "流水线模型", required = true) @@ -101,13 +103,211 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @QueryParam("pipelineId") pipelineId: String, - @ApiParam(value = "流水线模型", required = true) + @ApiParam( + value = "流水线模型", required = true, examples = Example( + value = [ + ExampleProperty( + mediaType = "如果我想更改流水线启动变量param的默认值为value2", + value = """ + { + "name": "更改流水线启动变量默认值", + "stages": [{ + "containers": [{ + "@type": "trigger", + "elements": [{ + "@type": "manualTrigger", + "...": "..." + }], + "params": [{ + "id": "param", + "required": true, + "type": "STRING", + "defaultValue": "value2", + "desc": "", + "readOnly": false + }] + }], + "...": "..." + }, { + "containers": [{}], + "...": "..." + }], + "...": "..." + } + """ + ), + ExampleProperty( + mediaType = "如果我想启用或是更改job互斥组配置", + value = """ + { + "stages": [{ + "containers": [{ + "@type": "trigger", + "...": "..." + }], + "...": "..." + }, { + "containers": [{ + "@type" : "vmBuild", + "name": "想要更改互斥组配置的job", + "elements": [{ + "...": "..." + }], + "...": "...", + "mutexGroup": { + "enable": true, + "mutexGroupName": "huchizu", + "queueEnable": true, + "timeout": 900, + "queue": 5 + } + }], + "...": "..." + }], + "...": "..." + } + """ + ), + ExampleProperty( + mediaType = "一般先通过接口(比如v3_app_pipeline_get)拿到编排，再根据自己的需求更改后上传更新", + value = """ + { + "name": "一个非常简单又完整的例子", + "desc": "", + "stages": [{ + "containers": [{ + "@type": "trigger", + "id": "0", + "name": "构建触发", + "elements": [{ + "@type": "manualTrigger", + "name": "手动触发", + "id": "T-1-1-1", + "canElementSkip": true, + "useLatestParameters": false, + "executeCount": 1, + "version": "1.*", + "classType": "manualTrigger", + "elementEnable": true, + "atomCode": "manualTrigger", + "taskAtom": "" + }], + "params": [], + "containerId": "0", + "containerHashId": "c-ccef587f17cd421a8a4e6aadc02777c6", + "executeCount": 0, + "matrixGroupFlag": false, + "classType": "trigger" + }], + "id": "stage-1", + "name": "stage-1", + "tag": ["28ee946a59f64949a74f3dee40a1bda4"], + "fastKill": false, + "finally": false + }, { + "containers": [{ + "@type": "vmBuild", + "id": "1", + "name": "构建环境-LINUX", + "elements": [{ + "@type": "linuxScript", + "name": "Bash", + "id": "e-efc1874f0cae44a0b56eba8b113b83f8", + "scriptType": "SHELL", + "script": "echo \"我只是为了测试\"", + "continueNoneZero": false, + "enableArchiveFile": false, + "archiveFile": "", + "additionalOptions": { + "enable": true, + "continueWhenFailed": false, + "manualSkip": false, + "retryWhenFailed": false, + "retryCount": 1, + "manualRetry": false, + "timeout": 900, + "runCondition": "PRE_TASK_SUCCESS", + "pauseBeforeExec": false, + "subscriptionPauseUser": "devops", + "otherTask": "", + "customVariables": [{ + "key": "param1", + "value": "" + }], + "customCondition": "", + "enableCustomEnv": false, + "customEnv": [{ + "key": "param1", + "value": "" + }] + }, + "executeCount": 1, + "version": "1.*", + "classType": "linuxScript", + "elementEnable": true, + "atomCode": "linuxScript", + "taskAtom": "" + }], + "baseOS": "LINUX", + "vmNames": [], + "maxQueueMinutes": 60, + "maxRunningMinutes": 900, + "buildEnv": {}, + "dispatchType": { + "buildType": "PUBLIC_DEVCLOUD", + "value": "tlinux3_ci", + "imageType": "BKSTORE", + "credentialId": "", + "credentialProject": "", + "imageCode": "tlinux3_ci", + "imageVersion": "2.*", + "imageName": "tlinux3-CI镜像", + "dockerBuildVersion": "tlinux3_ci", + "imagePublicFlag": false, + "imageRDType": "", + "recommendFlag": true + }, + "showBuildResource": false, + "enableExternal": false, + "containerId": "1", + "containerHashId": "c-758f14c0c5e644e1b70f1bf37a1cb5a5", + "executeCount": 0, + "jobId": "job_biS", + "matrixGroupFlag": false, + "nfsSwitch": false, + "classType": "vmBuild" + }], + "id": "stage-2", + "name": "stage-2", + "tag": ["28ee946a59f64949a74f3dee40a1bda4"], + "fastKill": false, + "finally": false, + "checkIn": { + "manualTrigger": false, + "timeout": 24 + }, + "checkOut": { + "manualTrigger": false, + "timeout": 24 + } + }], + "labels": [], + "instanceFromTemplate": false, + "pipelineCreator": "devops", + "events": {}, + "latestVersion": 6 + } + """ + ) + ] + ) + ) pipeline: Model ): Result @@ -124,7 +324,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -145,7 +345,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -166,7 +366,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -187,7 +387,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "流水线模型与设置", required = true) @@ -208,7 +408,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -232,7 +432,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID列表", required = true) @@ -246,7 +446,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "流水线模型", required = true) @@ -269,7 +469,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("第几页", required = false, defaultValue = "1") @@ -293,7 +493,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -314,7 +514,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -337,7 +537,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -358,7 +558,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = true) @@ -381,7 +581,7 @@ interface ApigwPipelineResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("搜索名称") diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPipelineViewResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPipelineViewResourceV4.kt new file mode 100644 index 00000000000..fe6b9774179 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwPipelineViewResourceV4.kt @@ -0,0 +1,320 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.api.apigw.v4 + +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.process.pojo.Pipeline +import com.tencent.devops.process.pojo.PipelineCollation +import com.tencent.devops.process.pojo.PipelineSortType +import com.tencent.devops.process.pojo.classify.PipelineNewView +import com.tencent.devops.process.pojo.classify.PipelineNewViewSummary +import com.tencent.devops.process.pojo.classify.PipelineViewForm +import com.tencent.devops.process.pojo.classify.PipelineViewId +import com.tencent.devops.process.pojo.classify.PipelineViewPipelinePage +import io.swagger.annotations.Api +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import io.swagger.annotations.Example +import io.swagger.annotations.ExampleProperty +import javax.ws.rs.Consumes +import javax.ws.rs.DELETE +import javax.ws.rs.GET +import javax.ws.rs.HeaderParam +import javax.ws.rs.POST +import javax.ws.rs.PUT +import javax.ws.rs.Path +import javax.ws.rs.PathParam +import javax.ws.rs.Produces +import javax.ws.rs.QueryParam +import javax.ws.rs.core.MediaType + +@Api(tags = ["OPENAPI_PIPELINE_VIEW_GROUP_V4"], description = "OPENAPI-流水线视图") +@Path("/{apigwType:apigw-user|apigw-app|apigw}/v4/projects/{projectId}/pipelineView") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface ApigwPipelineViewResourceV4 { + + @ApiOperation("用户获取视图流水线编排列表", tags = ["v4_user_pipeline_view_pipelines", "v4_app_pipeline_view_pipelines"]) + @GET + @Path("/listViewPipelines") + fun listViewPipelines( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + @ApiParam("第几页", required = false, defaultValue = "1") + @QueryParam("page") + page: Int?, + @ApiParam("每页多少条", required = false, defaultValue = "20") + @QueryParam("pageSize") + pageSize: Int?, + @ApiParam("流水线排序", required = false, defaultValue = "CREATE_TIME") + @QueryParam("sortType") + sortType: PipelineSortType? = PipelineSortType.CREATE_TIME, + @ApiParam("按流水线过滤", required = false) + @QueryParam("filterByPipelineName") + filterByPipelineName: String?, + @ApiParam("按创建人过滤", required = false) + @QueryParam("filterByCreator") + filterByCreator: String?, + @ApiParam("按标签过滤", required = false) + @QueryParam("filterByLabels") + filterByLabels: String?, + @ApiParam("按视图过滤", required = false) + @QueryParam("filterByViewIds") + filterByViewIds: String? = null, + @ApiParam("用户视图ID,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewId") + viewId: String?, + @ApiParam("用户视图名称,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewName") + viewName: String?, + @ApiParam("维度是否为项目,和viewName搭配使用", required = false) + @QueryParam("isProject") + isProject: Boolean?, + @ApiParam("排序规则", required = false) + @QueryParam("collation") + collation: PipelineCollation?, + @ApiParam("是否展示已删除流水线", required = false) + @QueryParam("showDelete") + showDelete: Boolean? = false + ): Result> + + @ApiOperation("获取视图列表", tags = ["v4_user_pipeline_view_list", "v4_app_pipeline_view_list"]) + @GET + @Path("/list") + fun listView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + @QueryParam("projected") + @ApiParam(value = "是否为项目流水线组 , 为空时不区分", required = false) + projected: Boolean? = null, + @QueryParam("viewType") + @ApiParam(value = "流水线组类型 , 1--动态, 2--静态 , 为空时不区分", required = false) + viewType: Int? = null + ): Result> + + @ApiOperation("添加视图", tags = ["v4_user_pipeline_view_create", "v4_app_pipeline_view_create"]) + @POST + @Path("") + fun addView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + @ApiParam( + "流水线视图创建模型", + examples = Example( + value = [ + ExampleProperty( + mediaType = "当使用静态分组(手动指定组内所包含的流水线)时," + + "仅需指定名称(name)、是否项目级别(projected)和分组类型(viewType:2)", + value = """ + { + "name": "我是分组名称", + "projected": true, + "viewType": 2 + } +""" + ), + ExampleProperty( + mediaType = "当使用静态分组时，同时传入几条流水线", + value = """ + { + "name": "我是分组名称", + "projected": true, + "viewType": 2, + "pipelineIds": [ + "p-xxx", + "p-xxx" + ] + } +""" + ), + ExampleProperty( + mediaType = "当使用动态分组(组内所包含的流水线根据设置的流水线名称、标签等属性去动态匹配)，" + + "需额外指定逻辑符(logic)、过滤规则(filters)", + value = """ + { + "name": "我是分组名称", + "projected": true, + "viewType": 1, + "logic": "AND", + "filters": [ + { + "@type": "filterByName", + "condition": "LIKE", + "pipelineName": "xxx" + }, + { + "@type": "filterByCreator", + "condition": "INCLUDE", + "userIds": [ + "xxx" + ] + } + ] + } + """ + ) + ] + ) + ) + pipelineView: PipelineViewForm + ): Result + + @ApiOperation("获取视图", tags = ["v4_user_pipeline_view_get", "v4_app_pipeline_view_get"]) + @GET + @Path("") + fun getView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String, + @ApiParam("用户视图ID,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewId") + viewId: String?, + @ApiParam("用户视图名称,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewName") + viewName: String?, + @ApiParam("维度是否为项目,和viewName搭配使用", required = false) + @QueryParam("isProject") + isProject: Boolean? + ): Result + + @ApiOperation("删除视图", tags = ["v4_user_pipeline_view_delete", "v4_app_pipeline_view_delete"]) + @DELETE + @Path("") + fun deleteView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + @ApiParam("用户视图ID,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewId") + viewId: String?, + @ApiParam("用户视图名称,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewName") + viewName: String?, + @ApiParam("维度是否为项目,和viewName搭配使用", required = false) + @QueryParam("isProject") + isProject: Boolean? + ): Result + + @ApiOperation("更改视图", tags = ["v4_user_pipeline_view_update", "v4_app_pipeline_view_update"]) + @PUT + @Path("") + fun updateView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String, + @ApiParam("用户视图ID,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewId") + viewId: String?, + @ApiParam("用户视图名称,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewName") + viewName: String?, + @ApiParam("维度是否为项目,和viewName搭配使用", required = false) + @QueryParam("isProject") + isProject: Boolean?, + @ApiParam( + "流水线视图更新模型", + examples = Example( + value = [ + ExampleProperty( + mediaType = "当给分组改名时," + + "仅需指定名称(name)、是否项目级别(projected)", + value = """ + { + "name": "我是分组名称", + "projected": true + } +""" + ), + ExampleProperty( + mediaType = "当使用静态分组时，新增流水线到分组，需要将原来的流水线加上，否则会被移除。", + value = """ + { + "name": "我是分组名称", + "projected": true, + "viewType": 2, + "pipelineIds": [ + "p-old1", + "p-old2", + "p-new" + ] + } +""" + ), + ExampleProperty( + mediaType = "当使用动态分组(组内所包含的流水线根据设置的流水线名称、标签等属性去动态匹配)，" + + "需指定逻辑符(logic)、过滤规则(filters)", + value = """ + { + "name": "我是分组名称", + "projected": true, + "viewType": 1, + "logic": "AND", + "filters": [ + { + "@type": "filterByName", + "condition": "LIKE", + "pipelineName": "xxx" + }, + { + "@type": "filterByCreator", + "condition": "INCLUDE", + "userIds": [ + "xxx" + ] + } + ] + } + """ + ) + ] + ) + ) + pipelineView: PipelineViewForm + ): Result +} diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwProjectResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwProjectResourceV4.kt index 991cf23f6bb..4bde66aee72 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwProjectResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwProjectResourceV4.kt @@ -90,7 +90,7 @@ interface ApigwProjectResourceV4 { @ApiParam("userId", required = true) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "项目信息", required = true) @@ -158,7 +158,7 @@ interface ApigwProjectResourceV4 { @ApiParam("项目名称或者项目英文名") @QueryParam("name") name: String, - @ApiParam("项目ID") + @ApiParam("项目ID(项目英文名)") @QueryParam("english_name") projectId: String? ): Result diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwQualityResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwQualityResourceV4.kt index 67a2a0ea7a3..b091247241f 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwQualityResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwQualityResourceV4.kt @@ -70,7 +70,7 @@ interface ApigwQualityResourceV4 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -94,7 +94,7 @@ interface ApigwQualityResourceV4 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -114,7 +114,7 @@ interface ApigwQualityResourceV4 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -137,7 +137,7 @@ interface ApigwQualityResourceV4 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -158,7 +158,7 @@ interface ApigwQualityResourceV4 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -170,7 +170,7 @@ interface ApigwQualityResourceV4 { @ApiParam("规则ID", required = false) @QueryParam("ruleHashId") ruleHashId: String?, - @ApiParam("状态", required = false) + @ApiParam("状态", required = false, type = "ENUM(PASS, FAIL)") @QueryParam("interceptResult") interceptResult: RuleInterceptResult?, @ApiParam("开始时间", required = false) @@ -200,7 +200,7 @@ interface ApigwQualityResourceV4 { @ApiParam(value = "apigw Type", required = true) @PathParam("apigwType") apigwType: String?, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryCommitResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryCommitResourceV4.kt index 5a836c7d8c7..d98cf33b42b 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryCommitResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryCommitResourceV4.kt @@ -65,7 +65,7 @@ interface ApigwRepositoryCommitResourceV4 { @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("流水线ID", required = false) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryResourceV4.kt index 25a658c45c8..09110cdbefb 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryResourceV4.kt @@ -39,6 +39,8 @@ import com.tencent.devops.repository.pojo.RepositoryInfo import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam +import io.swagger.annotations.Example +import io.swagger.annotations.ExampleProperty import javax.ws.rs.Consumes import javax.ws.rs.DELETE import javax.ws.rs.GET @@ -71,7 +73,7 @@ interface ApigwRepositoryResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("仓库类型", required = false) @@ -92,7 +94,7 @@ interface ApigwRepositoryResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "代码库模型", required = true) @@ -112,7 +114,7 @@ interface ApigwRepositoryResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("代码库哈希ID", required = true) @@ -133,13 +135,78 @@ interface ApigwRepositoryResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("代码库哈希ID", required = true) @QueryParam("repositoryHashId") repositoryHashId: String, - @ApiParam(value = "代码库模型", required = true) + @ApiParam( + value = "代码库模型", required = true, examples = Example( + value = [ + ExampleProperty( + mediaType = "如果我想通过oauth关联codeGit类型的代码库", + value = """ + { + "@type": "codeGit", + "aliasName": "devops/test", + "credentialId": "", + "projectName": "devops/test", + "url": "https://www.xxx.com/devops/test.git", + "authType": "OAUTH", + "svnType": "ssh", + "userName": "devops" + } + """ + ), + ExampleProperty( + mediaType = "如果我想关联TGIT类型的代码库，只能通过HTTP，需要使用凭据test", + value = """ + { + "@type": "codeTGit", + "aliasName": "devops/test", + "credentialId": "test", + "projectName": "devops/test", + "url": "https://git.tencent.com/devops/test.git", + "authType": "HTTPS", + "svnType": "ssh", + "userName": "devops" + } + """ + ), + ExampleProperty( + mediaType = "如果我想关联GitHub类型的代码库，只能通过Oauth", + value = """ + { + "@type": "github", + "aliasName": "Tencent/bk-ci", + "credentialId": "", + "projectName": "Tencent/bk-ci", + "url": "https://github.com/Tencent/bk-ci.git", + "authType": "OAUTH", + "svnType": "ssh", + "userName": "devops" + } + """ + ), + ExampleProperty( + mediaType = "如果我想关联P4类型的代码库，只能通过HTTP，需要使用凭据test", + value = """ + { + "@type": "codeP4", + "aliasName": "devops/test", + "credentialId": "test", + "projectName": "localhost:1666", + "url": "localhost:1666", + "authType": "HTTP", + "svnType": "ssh", + "userName": "devops" + } + """ + ) + ] + ) + ) repository: Repository ): Result } diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTemplateInstanceResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTemplateInstanceResourceV4.kt index dbec20b6777..89b9418ad37 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTemplateInstanceResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTemplateInstanceResourceV4.kt @@ -43,6 +43,8 @@ import com.tencent.devops.process.pojo.template.TemplateOperationRet import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam +import io.swagger.annotations.Example +import io.swagger.annotations.ExampleProperty import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.HeaderParam @@ -73,19 +75,59 @@ interface ApigwTemplateInstanceResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) @QueryParam("templateId") templateId: String, - @ApiParam("模板版本", required = true) + @ApiParam("模板版本（可通过v3_app_template_list接口获取）", required = true) @QueryParam("version") version: Long, @ApiParam("是否应用模板设置") @QueryParam("useTemplateSettings") useTemplateSettings: Boolean, - @ApiParam("创建实例", required = true) + @ApiParam( + "创建实例", required = true, examples = Example( + value = [ + ExampleProperty( + mediaType = "如果我想简单的实例化两条无启动变量的流水线1和2", + value = """ + [ + { + "pipelineName": "1", + "param": [] + }, + { + "pipelineName": "2", + "param": [] + } + ] + """ + ), + ExampleProperty( + mediaType = "如果我想实例化一条带启动变量param1的流水线3", + value = """ + [ + { + "pipelineName": "3", + "param": [ + { + "id": "param1", + "required": true, + "type": "STRING //可以是其他类型，以实际情况为准", + "defaultValue": "param1的值", + "desc": "", + "readOnly": false + } + ] + } + ] + """ + ) + ] + ) + ) instances: List ): TemplateOperationRet @@ -102,13 +144,13 @@ interface ApigwTemplateInstanceResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) @QueryParam("templateId") templateId: String, - @ApiParam("版本名", required = true) + @ApiParam("版本名（可通过v3_app_template_list接口获取）", required = true) @QueryParam("version") version: Long, @ApiParam("是否应用模板设置") @@ -118,7 +160,10 @@ interface ApigwTemplateInstanceResourceV4 { instances: List ): TemplateOperationRet - @ApiOperation("批量更新流水线模板实例", tags = ["v4_user_templateInstance_update", "v4_app_templateInstance_update"]) + @ApiOperation( + "批量更新流水线模板实例", + tags = ["v4_user_templateInstance_update_versionName", "v4_app_templateInstance_update_versionName"] + ) @PUT @Path("/update") fun updateTemplateInstances( @@ -131,7 +176,7 @@ interface ApigwTemplateInstanceResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) @@ -160,7 +205,7 @@ interface ApigwTemplateInstanceResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTemplateResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTemplateResourceV4.kt index 28613024f62..eb84b463416 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTemplateResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTemplateResourceV4.kt @@ -101,7 +101,7 @@ interface ApigwTemplateResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) @@ -149,7 +149,7 @@ interface ApigwTemplateResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模版类型", required = false) @@ -180,7 +180,7 @@ interface ApigwTemplateResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板", required = true) @@ -200,7 +200,7 @@ interface ApigwTemplateResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) @@ -221,7 +221,7 @@ interface ApigwTemplateResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) @@ -245,7 +245,7 @@ interface ApigwTemplateResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("模板ID", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTurboResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTurboResourceV4.kt index cd01a5dc08b..6fa9b1dbed5 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTurboResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwTurboResourceV4.kt @@ -57,7 +57,7 @@ interface ApigwTurboResourceV4 { @ApiOperation("获取方案列表", tags = ["v4_app_turbo_plan_list", "v4_user_turbo_plan_list"]) @Path("/projectId/{projectId}/turbo_plan_list") fun getTurboPlanByProjectIdAndCreatedDate( - @ApiParam(value = "项目id", required = true) + @ApiParam(value = "项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "开始日期", required = false) @@ -95,7 +95,7 @@ interface ApigwTurboResourceV4 { sortType: String?, @ApiParam(value = "编译加速历史请求数据信息", required = true) turboRecordModel: TurboRecordModel, - @ApiParam(value = "蓝盾项目id", required = true) + @ApiParam(value = "蓝盾项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "用户信息", required = true) @@ -110,7 +110,7 @@ interface ApigwTurboResourceV4 { @ApiParam(value = "方案id", required = true) @QueryParam("planId") planId: String, - @ApiParam(value = "蓝盾项目id", required = true) + @ApiParam(value = "蓝盾项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "用户信息", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/environment/ApigwEnvironmentResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/environment/ApigwEnvironmentResourceV4.kt index 677f4bbd125..c8e194ef7e4 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/environment/ApigwEnvironmentResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/environment/ApigwEnvironmentResourceV4.kt @@ -73,7 +73,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String ): Result> @@ -91,7 +91,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "环境信息", required = true) @@ -111,7 +111,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境 hashId", required = true) @@ -132,7 +132,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam(value = "节点列表", required = true) @@ -152,7 +152,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境 hashId", required = true) @@ -175,7 +175,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境 hashId", required = true) @@ -198,7 +198,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String ): Result> @@ -219,7 +219,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境名称(s)", required = true) @@ -242,7 +242,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境 hashId(s)", required = true) @@ -265,7 +265,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("节点 hashIds", required = true) @@ -288,7 +288,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("节点 hashIds", required = true) @@ -308,7 +308,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String ): Result> @@ -329,7 +329,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("节点 hashId", required = true) @@ -350,7 +350,7 @@ interface ApigwEnvironmentResourceV4 { @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) userId: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String, @ApiParam("环境 hashId", required = true) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/op/OpAppCodeProjectResource.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/op/OpAppCodeProjectResource.kt index a7e359e08f8..a7d80681dfc 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/op/OpAppCodeProjectResource.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/op/OpAppCodeProjectResource.kt @@ -61,7 +61,7 @@ interface OpAppCodeProjectResource { @ApiParam("appCode", required = true) @PathParam("appCode") appCode: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @QueryParam("projectId") projectId: String ): Result @@ -97,7 +97,7 @@ interface OpAppCodeProjectResource { @ApiParam("appCode", required = true) @PathParam("appCode") appCode: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String ): Result @@ -112,7 +112,7 @@ interface OpAppCodeProjectResource { @ApiParam("appCode", required = true) @PathParam("appCode") appCode: String, - @ApiParam("项目ID", required = true) + @ApiParam("项目ID(项目英文名)", required = true) @PathParam("projectId") projectId: String ): Result diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/op/OpSwaggerDocResource.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/op/OpSwaggerDocResource.kt new file mode 100644 index 00000000000..daaad4d71e7 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/op/OpSwaggerDocResource.kt @@ -0,0 +1,35 @@ +package com.tencent.devops.openapi.api.op + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.openapi.pojo.SwaggerDocResponse +import io.swagger.annotations.Api +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import javax.ws.rs.Consumes +import javax.ws.rs.DefaultValue +import javax.ws.rs.GET +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.QueryParam +import javax.ws.rs.core.MediaType + +@Api(tags = ["OP_APP_MANAGER_INFO"], description = "OP-AppCode管理员") +@Path("/op/swaggerDoc/") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Suppress("ALL") +interface OpSwaggerDocResource { + @ApiOperation("文档输出") + @GET + @Path("/init") + fun docInit( + @ApiParam("checkMetaData", required = false) + @QueryParam("checkMetaData") + @DefaultValue("false") + checkMetaData: Boolean, + @ApiParam("checkMDData", required = false) + @QueryParam("checkMDData") + @DefaultValue("true") + checkMDData: Boolean + ): Result> +} diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/ApigwMetricsSummary.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/ApigwMetricsSummary.kt new file mode 100644 index 00000000000..2fc213ec571 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/ApigwMetricsSummary.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.pojo + +import com.tencent.devops.metrics.pojo.vo.PipelineSumInfoVO +import com.tencent.devops.metrics.pojo.vo.ThirdPlatformOverviewInfoVO +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("流水线构建统计数据响应消息体") +data class ApigwMetricsSummary( + @ApiModelProperty("第三方汇总信息") + val overview: ThirdPlatformOverviewInfoVO?, + @ApiModelProperty("流水线汇总信息") + val sumInfo: PipelineSumInfoVO? +) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/AppCodeProjectResponse.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/AppCodeProjectResponse.kt index 63817a3db8b..a59fe46742c 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/AppCodeProjectResponse.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/AppCodeProjectResponse.kt @@ -35,7 +35,7 @@ data class AppCodeProjectResponse( val id: Long, @ApiModelProperty("appCode") val appCode: String, - @ApiModelProperty("项目ID") + @ApiModelProperty("项目ID(项目英文名)") val projectId: String, @ApiModelProperty("创建人") val creator: String?, diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/SwaggerDocParameterInfo.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/SwaggerDocParameterInfo.kt new file mode 100644 index 00000000000..65a43c322db --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/SwaggerDocParameterInfo.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.pojo + +import io.swagger.annotations.ApiModelProperty + +data class SwaggerDocParameterInfo( + @ApiModelProperty("是否可空") + val markedNullable: Boolean, + @ApiModelProperty("默认值") + val defaultValue: String? +) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewViewCreate.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/SwaggerDocResponse.kt similarity index 74% rename from src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewViewCreate.kt rename to src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/SwaggerDocResponse.kt index 413f65bbac2..33292fc0250 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewViewCreate.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/SwaggerDocResponse.kt @@ -24,21 +24,20 @@ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +package com.tencent.devops.openapi.pojo -package com.tencent.devops.process.pojo.classify - -import com.tencent.devops.process.pojo.classify.enums.Logic +import com.tencent.devops.openapi.utils.markdown.MarkdownElement import io.swagger.annotations.ApiModel import io.swagger.annotations.ApiModelProperty -@ApiModel("流水线视图创建模型") -data class PipelineNewViewCreate( - @ApiModelProperty("视图名称", required = false) - val name: String, - @ApiModelProperty("是否项目", required = false) - val projected: Boolean, - @ApiModelProperty("逻辑符", required = false) - val logic: Logic, - @ApiModelProperty("流水线视图过滤器列表", required = false) - val filters: List +@ApiModel("swagger文档") +data class SwaggerDocResponse( + @ApiModelProperty("请求path") + val path: String, + @ApiModelProperty("请求方法") + val httpMethod: String, + @ApiModelProperty("markdown文档") + val markdown: String?, + @ApiModelProperty("原始数据") + val metaData: List? ) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Code.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Code.kt new file mode 100644 index 00000000000..0bf6ecbf983 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Code.kt @@ -0,0 +1,17 @@ +package com.tencent.devops.openapi.utils.markdown + +import com.tencent.devops.openapi.utils.markdown.MarkdownCharacter.CODE_FILL + +class Code( + var language: String, + var body: String, + override val key: String = "" +) : MarkdownElement(key) { + companion object { + const val classType = "code" + } + + override fun toString(): String { + return "\n$CODE_FILL$language\n$body\n$CODE_FILL\n\n" + } +} diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Link.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Link.kt new file mode 100644 index 00000000000..b3a09fdddb8 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Link.kt @@ -0,0 +1,15 @@ +package com.tencent.devops.openapi.utils.markdown + +class Link( + var name: String, + var url: String, + override val key: String = "" +) : MarkdownElement(key) { + companion object { + const val classType = "link" + } + + override fun toString(): String { + return "[$name]($url)" + } +} diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/MarkdownCharacter.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/MarkdownCharacter.kt new file mode 100644 index 00000000000..d9e38c7cb2a --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/MarkdownCharacter.kt @@ -0,0 +1,10 @@ +package com.tencent.devops.openapi.utils.markdown + +object MarkdownCharacter { + const val SEPARATOR = "|" + const val FILL = '-' + const val WHITESPACE = " " + const val MIN_FILL = 3 + const val TEXT_FILL = '#' + const val CODE_FILL = "```" +} diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/MarkdownElement.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/MarkdownElement.kt new file mode 100644 index 00000000000..7dc431e7733 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/MarkdownElement.kt @@ -0,0 +1,15 @@ +package com.tencent.devops.openapi.utils.markdown + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "@type") +@JsonSubTypes( + JsonSubTypes.Type(value = Code::class, name = Code.classType), + JsonSubTypes.Type(value = Link::class, name = Link.classType), + JsonSubTypes.Type(value = Table::class, name = Table.classType), + JsonSubTypes.Type(value = Text::class, name = Text.classType) +) +open class MarkdownElement( + open val key: String +) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Table.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Table.kt new file mode 100644 index 00000000000..76ce7e7f244 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Table.kt @@ -0,0 +1,133 @@ +package com.tencent.devops.openapi.utils.markdown + +import com.tencent.devops.openapi.utils.markdown.MarkdownCharacter.FILL +import com.tencent.devops.openapi.utils.markdown.MarkdownCharacter.MIN_FILL +import com.tencent.devops.openapi.utils.markdown.MarkdownCharacter.SEPARATOR +import com.tencent.devops.openapi.utils.markdown.MarkdownCharacter.WHITESPACE + +class Table( + var header: TableRow = TableRow(), + var rows: List = emptyList(), + override val key: String = "" +) : MarkdownElement(key) { + companion object { + const val classType = "table" + } + + override fun toString(): String { + val body = StringBuffer() + if (rows.isEmpty()) { + body.append(Text(6, "无此参数", "")) + return body.toString() + } + body.append('\n') + val interval = getColumnWidths(rows.plus(header), MIN_FILL) + header.columns.tableJoinToString( + buffer = body, + separator = SEPARATOR, + prefix = SEPARATOR, + postfix = SEPARATOR + ) { index, element -> + WHITESPACE + element.padEnd(interval[index] ?: 0, ' ') + WHITESPACE + }.append('\n') + List(header.columns.size) { "-" }.tableJoinToString( + buffer = body, + separator = SEPARATOR, + prefix = SEPARATOR, + postfix = SEPARATOR + ) { index, element -> + WHITESPACE + element.padEnd(interval[index] ?: 0, FILL) + WHITESPACE + }.append('\n') + rows.forEach { + it.columns.tableJoinToString( + buffer = body, + separator = SEPARATOR, + prefix = SEPARATOR, + postfix = SEPARATOR + ) { index, element -> + WHITESPACE + element.padEnd(interval[index] ?: 0, ' ') + WHITESPACE + }.append('\n') + } + return body.append('\n').toString() + } + + fun setRow(vararg row: String): Table { + val columns = row.toList() + val new = rows.toMutableList() + new.removeIf { it.columns[0] == columns[0] } + new.add(0, TableRow(columns)) + rows = new + return this + } + + fun removeRow(key: String): Table { + val new = rows.toMutableList() + new.removeIf { it.columns[0] == key } + rows = new + return this + } + + fun checkLoadModel(loadModel: MutableList): Table { + rows.forEach { + if (it.columns[1].contains("](")) { + loadModel.addNoRepeat(it.columns[1].split("[")[1].split("]")[0]) + } + } + return this + } + + private fun getColumnWidths(rows: List, minimumColumnWidth: Int): Map { + val columnWidths: MutableMap = HashMap() + if (rows.isEmpty()) { + return columnWidths + } + for (columnIndex in 0 until rows[0].columns.size) { + columnWidths[columnIndex] = + getMaximumItemLength(rows, columnIndex, minimumColumnWidth) + } + return columnWidths + } + + private fun getMaximumItemLength(rows: List, columnIndex: Int, minimumColumnWidth: Int): Int { + var maximum = minimumColumnWidth + for (row in rows) { + if (row.columns.size < columnIndex + 1) { + continue + } + val value: Any = row.columns[columnIndex] + maximum = Math.max(value.toString().length, maximum) + } + return maximum + } + + private fun MutableList.addNoRepeat(text: T) { + if (text !in this) { + this.add(text) + } + } + + private fun Iterable.tableJoinToString( + buffer: A, + separator: CharSequence = ", ", + prefix: CharSequence = "", + postfix: CharSequence = "", + transform: ((Int, T) -> CharSequence) + ): A { + buffer.append(prefix) + var count = 0 + for (element in this) { + if (++count > 1) buffer.append(separator) + buffer.append(transform(count - 1, element)) + } + buffer.append(postfix) + return buffer + } +} + +class TableRow( + var columns: List = emptyList() +) { + constructor(vararg rows: Any?) : this() { + columns = rows.map { it?.toString() ?: "" } + } +} diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Text.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Text.kt new file mode 100644 index 00000000000..ae0fcc34f43 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/markdown/Text.kt @@ -0,0 +1,18 @@ +package com.tencent.devops.openapi.utils.markdown + +import com.tencent.devops.openapi.utils.markdown.MarkdownCharacter.TEXT_FILL +import com.tencent.devops.openapi.utils.markdown.MarkdownCharacter.WHITESPACE + +class Text( + var level: Int, + var body: String, + override val key: String = "" +) : MarkdownElement(key) { + companion object { + const val classType = "text" + } + + override fun toString(): String { + return WHITESPACE.padStart(level + 1, TEXT_FILL) + body + '\n' + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts b/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts index c2415644098..d41eea9bf07 100644 --- a/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts +++ b/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts @@ -38,4 +38,5 @@ dependencies { runtimeOnly("io.jsonwebtoken:jjwt-impl") runtimeOnly("io.jsonwebtoken:jjwt-jackson") api("org.springframework.boot:spring-boot-starter-aop") + testImplementation("org.mockito:mockito-inline") } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt index e9fb628a919..1b4d3548ed1 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt @@ -26,21 +26,27 @@ */ package com.tencent.devops.openapi.aspect +import com.tencent.devops.common.api.constant.HTTP_500 +import com.tencent.devops.common.api.exception.CustomException +import com.tencent.devops.common.api.exception.ParamBlankException import com.tencent.devops.common.api.exception.PermissionForbiddenException +import com.tencent.devops.common.api.exception.RemoteServiceException import com.tencent.devops.common.client.consul.ConsulConstants.PROJECT_TAG_REDIS_KEY import com.tencent.devops.common.redis.RedisOperation -import com.tencent.devops.openapi.IgnoreProjectId import com.tencent.devops.common.service.BkTag +import com.tencent.devops.openapi.IgnoreProjectId import com.tencent.devops.openapi.service.op.AppCodeService import com.tencent.devops.openapi.utils.ApiGatewayUtil import org.aspectj.lang.JoinPoint +import org.aspectj.lang.ProceedingJoinPoint +import org.aspectj.lang.annotation.Around import org.aspectj.lang.annotation.Aspect -import org.aspectj.lang.annotation.Before - import org.aspectj.lang.reflect.MethodSignature import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component +import javax.ws.rs.core.Response +import kotlin.reflect.jvm.kotlinFunction @Aspect @Component @@ -64,7 +70,6 @@ class ApiAspect( * @param jp */ // 所有controller包下面的所有方法的所有参数 - @Before("execution(* com.tencent.devops.openapi.resources.apigw..*.*(..))") @Suppress("ComplexMethod") fun beforeMethod(jp: JoinPoint) { if (!apiGatewayUtil.isAuth()) { @@ -118,15 +123,17 @@ class ApiAspect( } } - if (projectId != null && appCode != null && (apigwType == "apigw-app")) { - if (!appCodeService.validAppCode(appCode, projectId)) { + if (projectId != null) { + if (appCodeService.validProjectInfo(projectId) == null) { + appCodeService.invalidProjectInfo(projectId) + throw CustomException(Response.Status.NOT_FOUND, "ProjectId [$projectId] not find, please check it.") + } + + if (appCode != null && apigwType == "apigw-app" && !appCodeService.validAppCode(appCode, projectId)) { throw PermissionForbiddenException( message = "Permission denied: apigwType[$apigwType],appCode[$appCode],ProjectId[$projectId]" ) } - } - - if (projectId != null) { // openAPI 网关无法判别项目信息, 切面捕获project信息。 剩余一种URI内无${projectId}的情况,接口自行处理 val projectConsulTag = redisOperation.hget(PROJECT_TAG_REDIS_KEY, projectId) if (!projectConsulTag.isNullOrEmpty()) { @@ -135,12 +142,60 @@ class ApiAspect( } } + @Suppress("ComplexCondition") + @Around("execution(* com.tencent.devops.openapi.resources.apigw..*.*(..))") + fun aroundMethod(pdj: ProceedingJoinPoint): Any? { + val begin = System.currentTimeMillis() + val methodName = pdj.signature.name + beforeMethod(pdj) + + /*执行目标方法*/ + val res = try { + pdj.proceed() + } catch (error: RemoteServiceException) { + if (error.httpStatus >= HTTP_500) { + logger.error( + "openapi trigger remote service error,error code:${error.errorCode}| error info:${error.message}", + error + ) + } + logger.info( + "openapi trigger remote service failed,error code:${error.errorCode}| error info:${error.message}" + ) + throw error + } catch (ignored: ParamBlankException) { + logger.info("openapi check parameters error| error info:${ignored.message}") + throw CustomException(Response.Status.BAD_REQUEST, "参数校验失败: ${ignored.message}") + } catch (error: NullPointerException) { + // 如果在openapi层报NPE，一般是必填参数用户未传 + val parameterValue = pdj.args + val parameterMap = ((pdj.signature as MethodSignature).parameterNames zip parameterValue).toMap() + val parameters = (pdj.signature as MethodSignature).method.kotlinFunction?.parameters + + parameters?.forEach { kParameter -> + // 大多数调用失败都是参数缺失，所以进行null判断 + if (kParameter.name != null && // name为空的情况不需要判断 + !kParameter.type.isMarkedNullable && // 判断字段是否可空 + parameterMap.containsKey(kParameter.name) && // 检查参数集合中是否存在对应key，避免直接拿取到null + parameterMap[kParameter.name] == null // 判断用户传参是否为为null + ) { + throw CustomException(Response.Status.BAD_REQUEST, "参数校验失败: 请求参数${kParameter.name} 不能为空") + } + } + throw error + } finally { + afterMethod() + logger.info("$methodName 方法耗时${System.currentTimeMillis() - begin}毫秒") + } + + return res + } + /** * 后置增强：目标方法执行之前执行 * */ // 所有controller包下面的所有方法的所有参数 - @Before("execution(* com.tencent.devops.openapi.resources.apigw..*.*(..))") fun afterMethod() { // 删除线程ThreadLocal数据,防止线程池复用。导致流量指向被污染 bkTag.removeGatewayTag() diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/filter/impl/BlueKingApiFilter.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/filter/impl/BlueKingApiFilter.kt new file mode 100644 index 00000000000..7d165609467 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/filter/impl/BlueKingApiFilter.kt @@ -0,0 +1,233 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.filter.impl + +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_APP_CODE +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_APP_SECRET +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_USER_ID +import com.tencent.devops.common.api.exception.ErrorCodeException +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.service.utils.SpringContextUtil +import com.tencent.devops.common.web.RequestFilter +import com.tencent.devops.openapi.constant.OpenAPIMessageCode.ERROR_OPENAPI_JWT_PARSE_FAIL +import com.tencent.devops.openapi.filter.ApiFilter +import com.tencent.devops.openapi.utils.ApiGatewayPubFile +import com.tencent.devops.openapi.utils.ApiGatewayUtil +import io.jsonwebtoken.Jwts +import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.bouncycastle.openssl.PEMParser +import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Value +import java.io.ByteArrayInputStream +import java.io.InputStreamReader +import java.security.Security +import javax.ws.rs.container.ContainerRequestContext +import javax.ws.rs.container.PreMatching +import javax.ws.rs.core.Response +import javax.ws.rs.ext.Provider + +@Provider +@PreMatching +@RequestFilter +@Suppress("UNUSED") +class BlueKingApiFilter( + private val apiGatewayUtil: ApiGatewayUtil +) : ApiFilter { + + @Value("\${api.blueKing.enable:#{null}}") + private val apiFilterEnabled: Boolean? = false + + companion object { + private val logger = LoggerFactory.getLogger(BlueKingApiFilter::class.java) + private const val appCodeHeader = "app_code" + private const val appSecHeader = "app_secret" + private const val jwtHeader = "X-Bkapi-JWT" + } + + enum class ApiType(val startContextPath: String, val verify: Boolean) { + DEFAULT("/api/apigw/", true), + USER("/api/apigw-user/", true), + APP("/api/apigw-app/", true), + OP("/api/op/", false), + SWAGGER("/api/swagger.json", false); + + companion object { + fun parseType(path: String): ApiType? { + values().forEach { type -> + if (path.contains(other = type.startContextPath, ignoreCase = true)) { + return type + } + } + return null + } + } + } + + @Suppress("UNCHECKED_CAST", "ComplexMethod", "NestedBlockDepth", "ReturnCount") + override fun verifyJWT(requestContext: ContainerRequestContext): Boolean { + // path为为空的时候，直接退出 + val path = requestContext.uriInfo.requestUri.path + // 判断是否为合法的路径 + val apiType = ApiType.parseType(path) ?: return false + // 如果是op的接口访问直接跳过jwt认证 + if (!apiType.verify) return true + + logger.info("FILTER| url=$path") + val bkApiJwt = requestContext.getHeaderString(jwtHeader) + if (bkApiJwt.isNullOrBlank()) { + logger.error("Request bk api jwt is empty for ${requestContext.request}") + requestContext.abortWith( + Response.status(Response.Status.BAD_REQUEST) + .entity("Request bkapi jwt is empty.") + .build() + ) + return false + } + + val jwt = parseJwt(bkApiJwt) + logger.debug("Get the bkApiJwt header|X-Bkapi-JWT={}|jwt={}", bkApiJwt, jwt) + + // 验证应用身份信息 + if (jwt.contains("app")) { + val app = jwt["app"] as Map + // 应用身份登录 + if (app.contains(appCodeHeader)) { + val appCode = app[appCodeHeader]?.toString() + val verified = app["verified"].toString().toBoolean() + if (apiType == ApiType.APP && (appCode.isNullOrEmpty() || !verified)) { + return false + } else { + if (!appCode.isNullOrBlank()) { + // 将appCode头部置空 + requestContext.headers[AUTH_HEADER_DEVOPS_APP_CODE]?.set(0, null) + if (requestContext.headers[AUTH_HEADER_DEVOPS_APP_CODE] != null) { + requestContext.headers[AUTH_HEADER_DEVOPS_APP_CODE]?.set(0, appCode) + } else { + requestContext.headers.add(AUTH_HEADER_DEVOPS_APP_CODE, appCode) + } + } + } + } + } + // 在验证应用身份信息 + if (jwt.contains("user")) { + // 先做app的验证再做 + val user = jwt["user"] as Map + // 用户身份登录 + if (user.contains("username")) { + val username = user["username"]?.toString() ?: "" + val verified = user["verified"].toString().toBoolean() + // 名字为空或者没有通过认证的时候，直接失败 + if (username.isNotBlank() && verified) { + // 将user头部置空 + requestContext.headers[AUTH_HEADER_DEVOPS_USER_ID]?.set(0, null) + if (requestContext.headers[AUTH_HEADER_DEVOPS_USER_ID] != null) { + requestContext.headers[AUTH_HEADER_DEVOPS_USER_ID]?.set(0, username) + } else { + requestContext.headers.add(AUTH_HEADER_DEVOPS_USER_ID, username) + } + } else if (apiType == ApiType.USER) { + requestContext.abortWith( + Response.status(Response.Status.BAD_REQUEST) + .entity("Request don't has user's access_token.") + .build() + ) + return false + } + } + } + return true + } + + override fun filter(requestContext: ContainerRequestContext) { + if (apiFilterEnabled != true) { + return + } + if (!apiGatewayUtil.isAuth()) { + // 将query中的app_code和app_secret设置成头部 + setupHeader(requestContext) + } else { + // 验证通过 + if (!verifyJWT(requestContext)) { + requestContext.abortWith( + Response.status(Response.Status.BAD_REQUEST) + .entity("Devops OpenAPI Auth fail：user or app auth fail.") + .build() + ) + return + } + } + } + + private fun setupHeader(requestContext: ContainerRequestContext) { + requestContext.uriInfo?.pathParameters?.forEach { pathParam -> + if (pathParam.key == appCodeHeader && pathParam.value.isNotEmpty()) { + requestContext.headers[AUTH_HEADER_DEVOPS_APP_CODE]?.set(0, null) + if (requestContext.headers[AUTH_HEADER_DEVOPS_APP_CODE] != null) { + requestContext.headers[AUTH_HEADER_DEVOPS_APP_CODE]?.set(0, pathParam.value[0]) + } else { + requestContext.headers.add(AUTH_HEADER_DEVOPS_APP_CODE, pathParam.value[0]) + } + } else if (pathParam.key == appSecHeader && pathParam.value.isNotEmpty()) { + requestContext.headers[AUTH_HEADER_DEVOPS_APP_SECRET]?.set(0, null) + if (requestContext.headers[AUTH_HEADER_DEVOPS_APP_SECRET] != null) { + requestContext.headers[AUTH_HEADER_DEVOPS_APP_SECRET]?.set(0, pathParam.value[0]) + } else { + requestContext.headers.add(AUTH_HEADER_DEVOPS_APP_CODE, pathParam.value[0]) + } + } + } + } + + private fun parseJwt(bkApiJwt: String): Map { + var reader: PEMParser? = null + try { + val key = SpringContextUtil.getBean(ApiGatewayPubFile::class.java).getPubOuter().toByteArray() + + Security.addProvider(BouncyCastleProvider()) + val bais = ByteArrayInputStream(key) + reader = PEMParser(InputStreamReader(bais)) + val publicKeyInfo = reader.readObject() as SubjectPublicKeyInfo + val publicKey = JcaPEMKeyConverter().getPublicKey(publicKeyInfo) + val jwtParser = Jwts.parserBuilder().setSigningKey(publicKey).build() + val parse = jwtParser.parse(bkApiJwt) + logger.info("Get the parse body(${parse.body}) and header(${parse.header})") + return JsonUtil.toMap(parse.body) + } catch (ignored: Exception) { + logger.error("BKSystemErrorMonitor| Parse jwt failed.", ignored) + throw ErrorCodeException( + errorCode = ERROR_OPENAPI_JWT_PARSE_FAIL, + defaultMessage = "Parse jwt failed", + params = arrayOf(bkApiJwt) + ) + } finally { + reader?.close() + } + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/filter/impl/SampleApiFilter.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/filter/impl/SampleApiFilter.kt index 82431e9fbb1..21c412acc07 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/filter/impl/SampleApiFilter.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/filter/impl/SampleApiFilter.kt @@ -27,9 +27,6 @@ class SampleApiFilter constructor( private val apiFilterEnabled: Boolean? = false override fun verifyJWT(requestContext: ContainerRequestContext): Boolean { - if (apiFilterEnabled != true) { - return true - } val accessToken = requestContext.uriInfo.queryParameters.getFirst(API_ACCESS_TOKEN_PROPERTY) if (accessToken.isNullOrBlank()) { logger.warn("OPENAPI|verifyJWT accessToken is blank|" + @@ -58,6 +55,9 @@ class SampleApiFilter constructor( } override fun filter(requestContext: ContainerRequestContext) { + if (apiFilterEnabled != true) { + return + } if (!verifyJWT(requestContext)) { return } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v3/ApigwBuildResourceV3Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v3/ApigwBuildResourceV3Impl.kt index 9d0fdd5b10b..5e668490b1c 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v3/ApigwBuildResourceV3Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v3/ApigwBuildResourceV3Impl.kt @@ -131,7 +131,7 @@ class ApigwBuildResourceV3Impl @Autowired constructor( userId: String, projectId: String, pipelineId: String, - values: Map, + values: Map?, buildNo: Int? ): Result { logger.info("OPENAPI_BUILD_V3|$userId|start|$projectId|$pipelineId|$values|$buildNo") @@ -139,7 +139,7 @@ class ApigwBuildResourceV3Impl @Autowired constructor( userId = userId, projectId = projectId, pipelineId = pipelineId, - values = values, + values = values ?: emptyMap(), buildNo = buildNo, channelCode = apiGatewayUtil.getChannelCode(), startType = StartType.SERVICE @@ -175,8 +175,10 @@ class ApigwBuildResourceV3Impl @Autowired constructor( failedContainer: Boolean?, skipFailedTask: Boolean? ): Result { - logger.info("OPENAPI_BUILD_V3|$userId|retry|$projectId|$pipelineId|$buildId|$taskId|$failedContainer" + - "|$skipFailedTask") + logger.info( + "OPENAPI_BUILD_V3|$userId|retry|$projectId|$pipelineId|$buildId|$taskId|$failedContainer" + + "|$skipFailedTask" + ) return client.get(ServiceBuildResource::class).retry( userId = userId, projectId = projectId, @@ -219,8 +221,10 @@ class ApigwBuildResourceV3Impl @Autowired constructor( cancel: Boolean?, reviewRequest: StageReviewRequest? ): Result { - logger.info("OPENAPI_BUILD_V3|$userId|manual start stage|$projectId|$pipelineId|$buildId|$stageId|$cancel" + - "|$reviewRequest") + logger.info( + "OPENAPI_BUILD_V3|$userId|manual start stage|$projectId|$pipelineId|$buildId|$stageId|$cancel" + + "|$reviewRequest" + ) return client.get(ServiceBuildResource::class).manualStartStage( userId = userId, projectId = projectId, diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwBuildResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwBuildResourceV4Impl.kt index f9793620365..987e2a2ad19 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwBuildResourceV4Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwBuildResourceV4Impl.kt @@ -166,7 +166,7 @@ class ApigwBuildResourceV4Impl @Autowired constructor( userId: String, projectId: String, pipelineId: String, - values: Map, + values: Map?, buildNo: Int? ): Result { logger.info("OPENAPI_BUILD_V4|$userId|start|$projectId|$pipelineId|$values|$buildNo") @@ -174,7 +174,7 @@ class ApigwBuildResourceV4Impl @Autowired constructor( userId = userId, projectId = projectId, pipelineId = pipelineId, - values = values, + values = values ?: emptyMap(), buildNo = buildNo, channelCode = apiGatewayUtil.getChannelCode(), startType = StartType.SERVICE diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwMetricsResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwMetricsResourceV4Impl.kt new file mode 100644 index 00000000000..550c0550a72 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwMetricsResourceV4Impl.kt @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.resources.apigw.v4 + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.metrics.api.ServiceMetricsResource +import com.tencent.devops.metrics.pojo.vo.BaseQueryReqVO +import com.tencent.devops.openapi.api.apigw.v4.ApigwMetricsResourceV4 +import com.tencent.devops.openapi.pojo.ApigwMetricsSummary +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired + +@RestResource +class ApigwMetricsResourceV4Impl @Autowired constructor( + private val client: Client +) : ApigwMetricsResourceV4 { + + companion object { + private val logger = LoggerFactory.getLogger(ApigwMetricsResourceV4Impl::class.java) + } + + override fun getSummaryInfo(projectId: String, userId: String): Result { + logger.info( + "OPENAPI_METRICS_V4|$userId|get Summary by projectId id|$projectId" + ) + return Result( + ApigwMetricsSummary( + overview = client.get(ServiceMetricsResource::class).queryPipelineSummaryInfo( + projectId = projectId, + userId = userId, + startTime = null, + endTime = null + ).data, + sumInfo = client.get(ServiceMetricsResource::class).queryPipelineSumInfo( + projectId = projectId, + userId = userId, + baseQueryReq = BaseQueryReqVO() + ).data + ) + ) + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwPipelineResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwPipelineResourceV4Impl.kt index e0e432231cc..b63784c397d 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwPipelineResourceV4Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwPipelineResourceV4Impl.kt @@ -62,7 +62,8 @@ class ApigwPipelineResourceV4Impl @Autowired constructor( return client.get(ServicePipelineResource::class).status( userId = userId, projectId = projectId, - pipelineId = pipelineId + pipelineId = pipelineId, + channelCode = apiGatewayUtil.getChannelCode() ) } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwPipelineViewResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwPipelineViewResourceV4Impl.kt new file mode 100644 index 00000000000..408e6acaf04 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwPipelineViewResourceV4Impl.kt @@ -0,0 +1,172 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.resources.apigw.v4 + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.openapi.api.apigw.v4.ApigwPipelineViewResourceV4 +import com.tencent.devops.process.api.service.ServicePipelineViewResource +import com.tencent.devops.process.pojo.Pipeline +import com.tencent.devops.process.pojo.PipelineCollation +import com.tencent.devops.process.pojo.PipelineSortType +import com.tencent.devops.process.pojo.classify.PipelineNewView +import com.tencent.devops.process.pojo.classify.PipelineNewViewSummary +import com.tencent.devops.process.pojo.classify.PipelineViewForm +import com.tencent.devops.process.pojo.classify.PipelineViewId +import com.tencent.devops.process.pojo.classify.PipelineViewPipelinePage +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired + +@RestResource +class ApigwPipelineViewResourceV4Impl @Autowired constructor(private val client: Client) : + ApigwPipelineViewResourceV4 { + + override fun listViewPipelines( + userId: String, + projectId: String, + page: Int?, + pageSize: Int?, + sortType: PipelineSortType?, + filterByPipelineName: String?, + filterByCreator: String?, + filterByLabels: String?, + filterByViewIds: String?, + viewId: String?, + viewName: String?, + isProject: Boolean?, + collation: PipelineCollation?, + showDelete: Boolean? + ): Result> { + logger.info( + "OPENAPI_PIPELINE_VIEW_V4|$userId|list pipelines|" + + "$projectId|$page|$pageSize|$sortType|$filterByPipelineName|$filterByCreator|" + + "$filterByLabels|$filterByViewIds|$viewId|$viewName|$isProject|$collation|$showDelete|" + ) + return client.get(ServicePipelineViewResource::class).listViewPipelines( + userId = userId, + projectId = projectId, + page = page, + pageSize = pageSize, + sortType = sortType, + filterByPipelineName = filterByPipelineName, + filterByCreator = filterByCreator, + filterByLabels = filterByLabels, + filterByViewIds = filterByViewIds, + viewId = viewId, + viewName = viewName, + isProject = isProject, + collation = collation + ) + } + + override fun listView( + userId: String, + projectId: String, + projected: Boolean?, + viewType: Int? + ): Result> { + logger.info("OPENAPI_PIPELINE_VIEW_V4|$userId|list view|$projectId|$projected|$viewType") + return client.get(ServicePipelineViewResource::class).listView( + userId = userId, + projectId = projectId, + projected = projected, + viewType = viewType + ) + } + + override fun addView( + userId: String, + projectId: String, + pipelineView: PipelineViewForm + ): Result { + logger.info("OPENAPI_PIPELINE_VIEW_V4|$userId|add view|$projectId|$pipelineView") + return client.get(ServicePipelineViewResource::class).addView( + userId = userId, + projectId = projectId, + pipelineView = pipelineView + ) + } + + override fun getView( + userId: String, + projectId: String, + viewId: String?, + viewName: String?, + isProject: Boolean? + ): Result { + logger.info("OPENAPI_PIPELINE_VIEW_V4|$userId|get view|$projectId|$viewId|$viewName|$isProject") + return client.get(ServicePipelineViewResource::class).getView( + userId = userId, + projectId = projectId, + viewId = viewId, + viewName = viewName, + isProject = isProject + ) + } + + override fun deleteView( + userId: String, + projectId: String, + viewId: String?, + viewName: String?, + isProject: Boolean? + ): Result { + logger.info("OPENAPI_PIPELINE_VIEW_V4|$userId|delete view|$projectId|$viewId|$viewName|$isProject") + return client.get(ServicePipelineViewResource::class).deleteView( + userId = userId, + projectId = projectId, + viewId = viewId, + viewName = viewName, + isProject = isProject + ) + } + + override fun updateView( + userId: String, + projectId: String, + viewId: String?, + viewName: String?, + isProject: Boolean?, + pipelineView: PipelineViewForm + ): Result { + logger.info("OPENAPI_PIPELINE_VIEW_V4|$userId|update view|$projectId|$viewId|$viewName|$isProject") + return client.get(ServicePipelineViewResource::class).updateView( + userId = userId, + projectId = projectId, + viewId = viewId, + viewName = viewName, + isProject = isProject, + pipelineView = pipelineView + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(ApigwPipelineViewResourceV4Impl::class.java) + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/op/OpSwaggerDocResourceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/op/OpSwaggerDocResourceImpl.kt new file mode 100644 index 00000000000..2321b557692 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/op/OpSwaggerDocResourceImpl.kt @@ -0,0 +1,17 @@ +package com.tencent.devops.openapi.resources.op + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.openapi.api.op.OpSwaggerDocResource +import com.tencent.devops.openapi.pojo.SwaggerDocResponse +import com.tencent.devops.openapi.service.doc.DocumentService +import org.springframework.beans.factory.annotation.Autowired + +@RestResource +class OpSwaggerDocResourceImpl @Autowired constructor( + val docService: DocumentService +) : OpSwaggerDocResource { + override fun docInit(checkMetaData: Boolean, checkMDData: Boolean): Result> { + return Result(docService.docInit(checkMetaData, checkMDData)) + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/doc/DocumentService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/doc/DocumentService.kt new file mode 100644 index 00000000000..f365db71cdb --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/doc/DocumentService.kt @@ -0,0 +1,731 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.service.doc + +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_APP_CODE +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.util.FileUtil +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.openapi.pojo.SwaggerDocParameterInfo +import com.tencent.devops.openapi.pojo.SwaggerDocResponse +import com.tencent.devops.openapi.utils.markdown.Code +import com.tencent.devops.openapi.utils.markdown.Link +import com.tencent.devops.openapi.utils.markdown.MarkdownElement +import com.tencent.devops.openapi.utils.markdown.Table +import com.tencent.devops.openapi.utils.markdown.TableRow +import com.tencent.devops.openapi.utils.markdown.Text +import io.swagger.jaxrs.config.BeanConfig +import io.swagger.models.ArrayModel +import io.swagger.models.ComposedModel +import io.swagger.models.Model +import io.swagger.models.ModelImpl +import io.swagger.models.RefModel +import io.swagger.models.Response +import io.swagger.models.Swagger +import io.swagger.models.parameters.AbstractSerializableParameter +import io.swagger.models.parameters.BodyParameter +import io.swagger.models.parameters.HeaderParameter +import io.swagger.models.parameters.Parameter +import io.swagger.models.parameters.PathParameter +import io.swagger.models.parameters.QueryParameter +import io.swagger.models.parameters.RefParameter +import io.swagger.models.parameters.SerializableParameter +import io.swagger.models.properties.ArrayProperty +import io.swagger.models.properties.BooleanProperty +import io.swagger.models.properties.DoubleProperty +import io.swagger.models.properties.FloatProperty +import io.swagger.models.properties.IntegerProperty +import io.swagger.models.properties.LongProperty +import io.swagger.models.properties.MapProperty +import io.swagger.models.properties.ObjectProperty +import io.swagger.models.properties.PasswordProperty +import io.swagger.models.properties.Property +import io.swagger.models.properties.RefProperty +import io.swagger.models.properties.StringProperty +import io.swagger.models.properties.UUIDProperty +import org.springframework.beans.factory.annotation.Value +import org.springframework.stereotype.Service + +@Service +@Suppress("ComplexMethod") +class DocumentService { + @Value("\${spring.application.desc:#{null}}") + private val applicationDesc: String = "DevOps openapi Service" + + @Value("\${spring.application.version:#{null}}") + private val applicationVersion: String = "4.0" + + @Value("\${spring.application.packageName:#{null}}") + private val packageName: String = "com.tencent.devops.openapi" + + @Value("\${spring.application.name:#{null}}") + private val service: String = "openapi" + + private val onLoadTable = mutableMapOf() + + private val definitions = mutableMapOf() + + private lateinit var polymorphismMap: Map> + + /** + * swagger生成 markdown 文档。然后归档 + */ + @Suppress("NestedBlockDepth", "LongMethod") + fun docInit( + checkMetaData: Boolean, + checkMDData: Boolean, + polymorphism: Map> = emptyMap(), + outputPath: String? = null, + parametersInfo: Map>? = null + ): Map { + val response = mutableMapOf() + val swagger = loadSwagger() + definitions.putAll(swagger.definitions) + polymorphismMap = polymorphism + loadAllDefinitions(parametersInfo) + swagger.paths.forEach { (path, body) -> + // 遍历并生成每一path 不同 HttpMethod 下的文档 + body.operationMap.forEach { (httpMethod, operation) -> + val loadMarkdown = mutableListOf() + // 该path 需要组装的model + val onLoadModel = mutableListOf() + // 该path 已组装的model + val loadedModel = mutableListOf() +// loadMarkdown.add(Text(level = 1, body = "资源文档: ${operation.tags}", key = "resource_documentation")) + loadMarkdown.add(Text(level = 3, body = "请求方法/请求路径", key = "request_method")) + loadMarkdown.add(Text(level = 4, body = "$httpMethod $path", key = "http_method_path")) + loadMarkdown.add(Text(level = 3, body = "资源描述", key = "resource_description")) + loadMarkdown.add(Text(level = 4, body = operation.summary ?: "", key = "summary")) + loadMarkdown.add(Text(level = 3, body = "输入参数说明", key = "input_parameter_description")) + loadMarkdown.add(Text(level = 4, body = "Path参数", key = "path_parameter_title")) + loadMarkdown.add( + cacheOrLoad({ + Table( + header = TableRow("参数名称", "参数类型", "必须", "参数说明", "默认值"), + rows = parseParameters(operation.parameters.filterIsInstance()), + key = "path_parameter" + ).checkLoadModel(onLoadModel) + }, path + httpMethod + "path") + ) + loadMarkdown.add(Text(4, "Query参数", "query_parameter_title")) + loadMarkdown.add( + cacheOrLoad({ + Table( + header = TableRow("参数名称", "参数类型", "必须", "参数说明", "默认值"), + rows = parseParameters(operation.parameters.filterIsInstance()), + "query_parameter" + ).checkLoadModel(onLoadModel) + }, path + httpMethod + "query") + ) + loadMarkdown.add(Text(4, "Header参数", "header_parameter_title")) + loadMarkdown.add( + cacheOrLoad({ + Table( + header = TableRow("参数名称", "参数类型", "必须", "参数说明", "默认值"), + rows = parseParameters(operation.parameters.filterIsInstance()), + "header_parameter" + ).checkLoadModel(onLoadModel) + .setRow(AUTH_HEADER_USER_ID, "string", "应用态必填、用户态不填", "用户名", "{X-DEVOPS-UID}") + .setRow("Content-Type", "string", "是", "", "application/json") + .removeRow(AUTH_HEADER_DEVOPS_APP_CODE) + }, path + httpMethod + "header") + ) + loadMarkdown.add(Text(4, "Body参数", "body_parameter_title")) + loadMarkdown.add( + cacheOrLoad({ + Table( + header = TableRow("参数名称", "参数类型", "必须", "参数说明", "默认值"), + rows = parseParameters(operation.parameters.filterIsInstance()), + "body_parameter" + ).checkLoadModel(onLoadModel) + }, path + httpMethod + "body") + ) + loadMarkdown.add(Text(4, "响应参数", "response_parameter_title")) + loadMarkdown.add( + cacheOrLoad({ + Table( + header = TableRow("HTTP代码", "参数类型", "说明"), + rows = parseResponse(operation.responses), + "response_parameter" + ).checkLoadModel(onLoadModel) + }, path + httpMethod + "response") + ) + // payload 样例 + loadMarkdown.addAll( + parsePayloadExample( + operation.parameters.filterIsInstance() + ) + ) + loadMarkdown.add(Text(3, "Curl 请求样例", "curl_request_sample_title")) + loadMarkdown.add( + Code( + "Json", + parseCurlExample( + httpMethod = httpMethod.name, + query = cacheOrLoad({ null }, path + httpMethod + "query"), + header = cacheOrLoad({ null }, path + httpMethod + "header") + ), + "curl_request_sample" + ) + ) + // 组装请求样例 + loadMarkdown.addAll( + parseRequestExampleJson( + httpMethod.name, + operation.parameters.filterIsInstance() + ) + ) + // 组装返回样例 + loadMarkdown.addAll( + parseResponseExampleJson( + operation.responses + ) + ) + loadMarkdown.add(Text(3, "相关模型数据", "all_model_data")) + // 组装所有已使用的模型 + loadMarkdown.addAll(parseAllModel(onLoadModel, loadedModel)) + operation.tags.forEach { tag -> + response[tag] = SwaggerDocResponse( + path = path, + httpMethod = httpMethod.name, + markdown = if (checkMDData) loadMarkdown.joinToString(separator = "") else null, + metaData = if (checkMetaData) loadMarkdown else null + ) + if (!outputPath.isNullOrBlank()) { + FileUtil.outFile(outputPath, "$tag.md", loadMarkdown.joinToString(separator = "")) + } + } + } + } + if (!outputPath.isNullOrBlank()) { + FileUtil.outFile(outputPath, "all.json", JsonUtil.toJson(response)) + } + onLoadTable.clear() + definitions.clear() + return response + } + + private fun cacheOrLoad(func: () -> Table?, key: String): Table { + val cache = onLoadTable[key] + if (cache != null) { + return cache + } + val load = func() ?: return Table() + onLoadTable[key] = load + return load + } + + private fun loadAllDefinitions(parametersInfo: Map>? = null) { + definitions.forEach { (key, model) -> + cacheOrLoad({ + val tableRows = mutableListOf() + loadModelDefinitions(model, tableRows) + tableRows.forEach { table -> + val reflectInfo = parametersInfo?.get(key)?.get(table.columns[0]) + if (reflectInfo != null) { + val column = table.columns.toMutableList() + column[2] = if (reflectInfo.markedNullable.not()) "是" else "否" + column[4] = if (reflectInfo.markedNullable) reflectInfo.defaultValue ?: "" else column[4] + table.columns = column + } + } + Table( + header = TableRow("参数名称", "参数类型", "必须", "参数说明", "默认值"), + rows = tableRows, + key = "model_$key" + ) + }, key) + } + } + + private fun parseAllModel( + modelList: List, + loadedModel: MutableList + ): List { + val markdownElement = mutableListOf() + modelList.forEach { + if (it in loadedModel) return@forEach + val onLoadModel = mutableListOf() + val model = cacheOrLoad({ + val tableRows = mutableListOf() + definitions[it]?.let { model -> loadModelDefinitions(model, tableRows) } + Table( + header = TableRow("参数名称", "参数类型", "必须", "参数说明", "默认值"), + rows = tableRows, + key = "model_$it" + ) + }, it).apply { + if (it in polymorphismMap) { + this.setRow( + (definitions[it] as ModelImpl).discriminator, + "string", + "是", + "用于指定实现某一多态类, 可选${polymorphismMap[it]?.keys},具体实现见下方", + "" + ) + } + }.checkLoadModel(onLoadModel) + markdownElement.add(Text(level = 4, body = it, key = "model_${it}_title")) + markdownElement.add(model) + loadedModel.add(it) + + // 多态类展示 + polymorphismMap[it]?.forEach { (child, value) -> + val discriminator = (definitions[it] as ModelImpl).discriminator + val childModel = cacheOrLoad({ null }, child) + .setRow(discriminator, "string", "必须是[$value]", "多态类实现", value) + .checkLoadModel(onLoadModel) + markdownElement.add(Text(4, child, "polymorphism_model_${child}_title")) + markdownElement.add( + Text( + level = 0, body = "*多态基类 <$it> 的实现处, 其中当字段 $discriminator = [$value] 时指定为该类实现*", + key = "polymorphism_model_$child" + ) + ) + markdownElement.add(Text(level = 0, body = "", key = "")) + markdownElement.add(childModel) + loadedModel.add(child) + } + + if (onLoadModel.isNotEmpty()) { + markdownElement.addAll(parseAllModel(onLoadModel, loadedModel)) + } + } + return markdownElement + } + + private fun parseResponseExampleJson(responses: Map): List { + val markdownElement = mutableListOf() + responses.forEach { (httpStatus, response) -> + val loadJson = mutableMapOf() + loadModelJson(response.responseSchema, loadJson) + markdownElement.add( + Text(level = 3, body = "$httpStatus 返回样例", key = "${httpStatus}_return_example_title") + ) + markdownElement.add( + Code(language = "Json", body = JsonUtil.toJson(loadJson), key = "${httpStatus}_return_example") + ) + } + return markdownElement + } + + private fun parsePayloadExample(body: List): List { + if (body.getOrNull(0)?.examples?.isEmpty() != false) return emptyList() + val res = mutableListOf() + res.add(Text(level = 3, body = "Request Payload 举例", key = "Payload_request_sample_title")) + res.add( + Text( + level = 0, + body = "**注意: 确保 header 中存在 Content-Type: application/json ,否则请求返回415错误码**", + key = "Payload_request_sample_explain" + ) + ) + body[0].examples.forEach { (texplain, jsonSimple) -> + res.add( + Text( + level = 4, + body = "< $texplain >, 那么请求应该为:", + key = "Payload_request_sample_title_$texplain" + ) + ) + val jsonString = try { + JsonUtil.toJson(JsonUtil.to(jsonSimple)) + } catch (e: Throwable) { + jsonSimple + } + res.add(Code(language = "Json", body = jsonString, key = "Payload_request_sample_json_$texplain")) + } + return res + } + + private fun parseRequestExampleJson(httpMethod: String, body: List): List { + if (body.isEmpty()) return emptyList() + val schema = body[0].schema + val outJson: Any = when (schema) { + is ComposedModel -> { + val loadJson = mutableMapOf() + schema.allOf?.forEach { + loadModelJson(it, loadJson) + } + loadJson + } + is ModelImpl -> { + val loadJson = mutableMapOf() + schema.properties?.forEach { (key, property) -> + loadJson[key] = loadPropertyJson(property) + } + loadJson + } + is RefModel -> { + val loadJson = mutableMapOf() + loadModelJson(schema, loadJson) + loadJson + } + is ArrayModel -> { + val loadJson = mutableListOf() + loadJson.add(loadPropertyJson(schema.items)) + loadJson + } + else -> { + emptyMap() + } + } + return listOf( + Text(level = 3, body = "$httpMethod 请求样例", key = "${httpMethod}_request_sample_title"), + Code(language = "Json", body = JsonUtil.toJson(outJson), key = "${httpMethod}_request_sample") + ) + } + + private fun parseCurlExample(httpMethod: String, query: Table, header: Table): String { + val queryString = query.rows.takeIf { it.isNotEmpty() }?.joinToString(prefix = "?", separator = "&") { + "${it.columns[0]}={${it.columns[0]}}" + } ?: "" + val headerString = header.rows.takeIf { it.isNotEmpty() }?.joinToString(prefix = "\\\n", separator = "\\\n") { + "-H '${it.columns[0]}: ${it.columns[4]}' " + } ?: "" + return "curl -X ${httpMethod.toUpperCase()} '[请替换为上方API地址栏请求地址]$queryString' $headerString" + } + + private fun parseResponse(responses: Map): List { + val tableRow = mutableListOf() + responses.forEach { (httpStatus, response) -> + tableRow.addNoRepeat( + TableRow(httpStatus, loadModelType(response.responseSchema), response.description) + ) + } + return tableRow + } + + private fun parseParameters(parameters: List): List { + val tableRow = mutableListOf() + parameters.forEach { + when (it) { + is BodyParameter -> { + tableRow.addNoRepeat( + TableRow( + it.name, + loadModelType(it.schema), + if (it.required) "是" else "否", + it.description, + "" + ) + ) + } + is AbstractSerializableParameter<*> -> { + tableRow.addNoRepeat( + TableRow( + it.name, + loadSerializableParameter(it), + if (it.required) "是" else "否", + it.description, + it.defaultValue + ) + ) + } +// is PathParameter -> {} +// is QueryParameter -> {} + is RefParameter -> { + tableRow.addNoRepeat( + TableRow( + it.name, + Link(it.originalRef, '#' + it.originalRef).toString(), + if (it.required) "是" else "否", + it.description, + "" + ) + ) + } + else -> {} + } + } + return tableRow + } + + private fun loadSwagger(): Swagger { + val bean = BeanConfig().apply { + title = applicationDesc + version = applicationVersion + resourcePackage = packageName + scan = true + basePath = "/$service/api" + } + return bean.swagger + } + + private fun loadSerializableParameter(parameter: SerializableParameter): String { + return when (parameter.type) { + "string" -> { + val enum = parameter.enumValue + if (enum.isNullOrEmpty()) { + parameter.type + } else { + val str = enum.toEnumString() + "ENUM($str)" + } + } + "array" -> { + "List<" + loadPropertyType(parameter.items) + ">" + } + "integer" -> { + when (parameter.format) { + "int32" -> "Int" + "int64" -> "Long" + else -> "integer" + } + } + else -> parameter.type + } + } + + private fun loadModelDefinitions( + model: Model, + tableRow: MutableList + ) { + when (model) { + is ComposedModel -> { + model.allOf?.forEach { + loadModelDefinitions(it, tableRow) + } +// +// // 初始化多态类 +// if (model.parent is RefModel && model.child is ModelImpl) { +// val ref = model.parent as RefModel +// val impl = model.child as ModelImpl +// polymorphismMap[ref.originalRef].apply { +// if (this == null) { +// polymorphismMap[ref.originalRef] = mutableListOf(impl.name) +// } else { +// this.addNoRepeat(impl.name) +// } +// } +// } + } + + is ModelImpl -> { + model.properties?.forEach { (key, property) -> + tableRow.addNoRepeat( + TableRow( + key, + loadPropertyType(property), + if (property.required) "是" else "否", + loadDescriptionInfo(property), + loadPropertyDefault(property) + ) + ) + } + } + + is RefModel -> { + tableRow.addAll( + cacheOrLoad( + { + val table = mutableListOf() + definitions[model.originalRef]?.let { + loadModelDefinitions(it, table) + } + Table( + header = TableRow("参数名称", "参数类型", "必须", "参数说明"), + rows = table, + key = "model_${model.originalRef}" + ) + }, model.originalRef + ).rows + ) + } + else -> {} + } + } + + private fun loadDescriptionInfo(property: Property?): String { + if (property == null) return "" + val res = StringBuffer() + if (property.readOnly == true) { + res.append("(该字段只读)") + } + res.append(property.description ?: "") + return res.toString() + } + + private fun loadModelType(model: Model?): String { + if (model == null) return "" + return when (model) { +// is ComposedModel -> {} + is ModelImpl -> { + "Map" + } + is RefModel -> { + Link(model.originalRef, '#' + model.originalRef).toString() + } + is ArrayModel -> { + "List<" + loadPropertyType(model.items) + ">" + } + else -> { + "parse error" + } + } + } + + private fun loadModelJson(model: Model?, loadJson: MutableMap) { + if (model == null) return + when (model) { + is ComposedModel -> { + model.allOf?.forEach { + loadModelJson(it, loadJson) + } + } + is ModelImpl -> { + if (model.discriminator != null) { + loadJson[model.discriminator] = "string" + } + model.properties?.forEach { (key, property) -> + loadJson[key] = loadPropertyJson(property) + } + } + is RefModel -> { + definitions[model.originalRef]?.let { loadModelJson(it, loadJson) } + } + else -> {} + } + } + + private fun loadPropertyJson(property: Property): Any { + return when (property) { + is RefProperty -> { + val loadJson = mutableMapOf() + definitions[property.originalRef]?.let { loadModelJson(it, loadJson) } + loadJson + } + // swagger无法获取到map的key类型 + is MapProperty -> { + mapOf("string" to loadPropertyJson(property.additionalProperties)) + } + is ObjectProperty -> { + "Any 任意类型，参照实际请求或返回" + } + is ArrayProperty -> { + listOf(loadPropertyJson(property.items)) + } + is StringProperty -> { + if (property.enum == null) { + property.type + } else { + "enum" + } + } + is BooleanProperty -> false + is IntegerProperty -> 0 + is LongProperty -> 0L + is DoubleProperty -> 0.0 + is FloatProperty -> 0f + else -> { + property.type + } + } + } + + private fun loadPropertyDefault(property: Property?): String? { + if (property == null) return null + return when (property) { + is BooleanProperty -> { + property.default?.toString() + } + is DoubleProperty -> { + property.default?.toString() + } + is FloatProperty -> { + property.default?.toString() + } + is IntegerProperty -> { + property.default?.toString() + } + is LongProperty -> { + property.default?.toString() + } + is StringProperty -> { + property.default?.toString() + } + is PasswordProperty -> { + property.default?.toString() + } + is UUIDProperty -> { + property.default?.toString() + } + else -> { + null + } + } + } + + private fun loadPropertyType(property: Property?): String { + if (property == null) return "" + return when (property) { + is RefProperty -> { + Link(property.originalRef, '#' + property.originalRef).toString() + } + // swagger无法获取到map的key类型 + is MapProperty -> { + "Map" + } + is ObjectProperty -> { + "Any" + } + is ArrayProperty -> { + "List<" + loadPropertyType(property.items) + ">" + } + is StringProperty -> { + if (property.enum.isNullOrEmpty()) { + property.type + } else { + "ENUM(" + property.enum.toEnumString() + ")" + } + } + else -> { + property.type + } + } + } + + private fun List.toEnumString(): String { + var result = "" + this.forEachIndexed { index, any -> + if (index == this.size - 1) { + result = "$result$any" + return@forEachIndexed + } + result = "$result$any, " + } + return result + } + + private fun MutableList.addNoRepeat(table: TableRow) { + val row = this.find { it.columns[0] == table.columns[0] } + if (row != null) { + this.remove(row) + } + this.add(table) + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/op/AppCodeService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/op/AppCodeService.kt index 17ac37a7817..8132e4b9c54 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/op/AppCodeService.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/op/AppCodeService.kt @@ -60,10 +60,10 @@ class AppCodeService( override fun load(appCode: String): Map { return try { val projectMap = getAppCodeProject(appCode) - logger.info("appCode[$appCode] openapi projectMap:$projectMap.") + logger.info("appCode[$appCode] openapi projectMap|$projectMap.") projectMap } catch (t: Throwable) { - logger.info("appCode[$appCode] failed to get projectMap.") + logger.warn("appCode[$appCode] failed to get projectMap.", t) mutableMapOf() } } @@ -78,10 +78,10 @@ class AppCodeService( override fun load(appCode: String): Pair { return try { val appCodeGroup = getAppCodeGroup(appCode) - logger.info("appCode[$appCode] openapi appCodeGroup:$appCodeGroup.") + logger.info("appCode[$appCode] openapi appCodeGroup|$appCodeGroup.") Pair(appCode, appCodeGroup) } catch (t: Throwable) { - logger.info("appCode[$appCode] failed to get appCodeGroup.") + logger.warn("appCode[$appCode] failed to get appCodeGroup.", t) Pair(appCode, null) } } @@ -96,10 +96,10 @@ class AppCodeService( override fun load(projectId: String): Pair { return try { val projectInfo = client.get(ServiceProjectResource::class).get(projectId).data - logger.info("projectId[$projectId] openapi projectInfo:$projectInfo.") + logger.info("projectId[$projectId] openapi projectInfo|$projectInfo.") Pair(projectId, projectInfo) } catch (t: Throwable) { - logger.info("projectId[projectIdappCode] failed to get projectInfo.") + logger.warn("projectId[$projectId] failed to get projectInfo.", t) Pair(projectId, null) } } @@ -131,9 +131,13 @@ class AppCodeService( } } + fun validProjectInfo(projectId: String) = projectInfoCache.get(projectId).second + + fun invalidProjectInfo(projectId: String) = projectInfoCache.invalidate(projectId) + fun validAppCode(appCode: String, projectId: String): Boolean { val appCodeProject = appCodeProjectCache.get(appCode) - logger.info("appCode[$appCode] projectId[$projectId] openapi appCodeProjectCache:$appCodeProject.") + logger.info("appCode[$appCode] projectId[$projectId] openapi appCodeProjectCache|$appCodeProject.") if (appCodeProject.isNotEmpty()) { val projectId2 = appCodeProject[projectId] if (projectId2 != null && projectId2.isNotBlank()) { @@ -143,26 +147,29 @@ class AppCodeService( } logger.info("appCode[$appCode] projectId[$projectId] openapi appCodeProjectCache no matched.") val appCodeGroup = appCodeGroupCache.get(appCode).second - logger.info("appCode[$appCode] projectId[$projectId] openapi appCodeGroupCache:$appCodeGroup.") + logger.info("appCode[$appCode] projectId[$projectId] openapi appCodeGroupCache|$appCodeGroup.") if (appCodeGroup != null) { val projectInfo = projectInfoCache.get(projectId).second - logger.info("appCode[$appCode] projectId[$projectId] openapi appCodeGroupCache projectInfo:$projectInfo.") + logger.info("appCode[$appCode] projectId[$projectId] openapi appCodeGroupCache projectInfo|$projectInfo.") if (projectInfo != null) { if (appCodeGroup.centerId != null && projectInfo.centerId != null && - appCodeGroup.centerId.toString() == projectInfo.centerId) { + appCodeGroup.centerId.toString() == projectInfo.centerId + ) { logger.info("appCode[$appCode] projectId[$projectId] openapi appCodeGroupCache centerId matched.") return true } if (appCodeGroup.deptId != null && projectInfo.deptId != null && - appCodeGroup.deptId.toString() == projectInfo.deptId) { + appCodeGroup.deptId.toString() == projectInfo.deptId + ) { logger.info("appCode[$appCode] projectId[$projectId] openapi appCodeGroupCache deptId matched.") return true } if (appCodeGroup.bgId != null && projectInfo.bgId != null && - appCodeGroup.bgId.toString() == projectInfo.bgId) { + appCodeGroup.bgId.toString() == projectInfo.bgId + ) { logger.info("appCode[$appCode] projectId[$projectId] openapi appCodeGroupCache bgId matched.") return true } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/ApiGatewayPubFile.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/ApiGatewayPubFile.kt index d677a833381..dff646e41eb 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/ApiGatewayPubFile.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/utils/ApiGatewayPubFile.kt @@ -48,11 +48,7 @@ class ApiGatewayPubFile { @Value("\${api.gateway.pub.file.outer:#{null}}") private val pubFileOuter: String? = null - @Value("\${api.gateway.pub.file.inner:#{null}}") - private val pubFileInner: String? = null - private var pubOuter: String? = null - private var pubInner: String? = null fun getPubOuter(): String { if (pubOuter == null) { @@ -97,48 +93,4 @@ class ApiGatewayPubFile { return pubOuter!! } - - fun getPubInner(): String { - if (pubInner == null) { - synchronized(this) { - if (pubInner != null) { - return pubInner!! - } - if (pubFileInner == null) { - throw InvalidConfigException( - message = "Api gateway pub file is not settle", - errorCode = ERROR_OPENAPI_APIGW_PUBFILE_NOT_SETTLE - ) - } - - val file = File(pubFileInner) - if (!file.exists()) { - throw InvalidConfigException( - message = "The pub file (${file.absolutePath}) is not exist", - errorCode = ERROR_OPENAPI_APIGW_PUBFILE_NOT_EXIST, - params = arrayOf(file.absolutePath) - ) - } - pubInner = file.readText() - if (pubInner == null) { - throw InvalidConfigException( - message = "Can't read the pub content from ${file.absolutePath}", - errorCode = ERROR_OPENAPI_APIGW_PUBFILE_READ_ERROR, - params = arrayOf(file.absolutePath) - ) - } - - if (pubInner!!.trim().isEmpty()) { - throw InvalidConfigException( - message = "The pub file is empty from ${file.absolutePath}", - errorCode = ERROR_OPENAPI_APIGW_PUBFILE_CONTENT_EMPTY, - params = arrayOf(file.absolutePath) - ) - } - logger.info("Get the pub($pubInner) from ${file.absolutePath}") - } - } - - return pubInner!! - } } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/test/kotlin/com/tencent/devops/openapi/service/doc/DocumentServiceTest.kt b/src/backend/ci/core/openapi/biz-openapi/src/test/kotlin/com/tencent/devops/openapi/service/doc/DocumentServiceTest.kt new file mode 100644 index 00000000000..c0a8e5934a2 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/test/kotlin/com/tencent/devops/openapi/service/doc/DocumentServiceTest.kt @@ -0,0 +1,213 @@ +package com.tencent.devops.openapi.service.doc + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.tencent.devops.common.web.JerseyConfig +import com.tencent.devops.openapi.pojo.SwaggerDocParameterInfo +import io.swagger.annotations.ApiModel +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Mockito +import org.mockito.creation.instance.InstantiationException +import org.mockito.internal.configuration.plugins.Plugins +import org.mockito.plugins.MemberAccessor +import org.mockito.plugins.MemberAccessor.ConstructionDispatcher +import org.reflections.Reflections +import org.reflections.scanners.Scanners +import org.reflections.util.ClasspathHelper +import org.reflections.util.ConfigurationBuilder +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.junit.jupiter.SpringExtension +import kotlin.jvm.internal.DefaultConstructorMarker +import kotlin.reflect.KFunction +import kotlin.reflect.KType +import kotlin.reflect.full.memberProperties +import kotlin.reflect.jvm.isAccessible +import kotlin.reflect.jvm.javaConstructor +import kotlin.reflect.jvm.javaType + +@ExtendWith(SpringExtension::class) +@SpringBootTest(classes = [JerseyConfig::class, DocumentService::class]) +class DocumentServiceTest @Autowired constructor( + private val document: DocumentService +) { + private val mockitoConstruction = ThreadLocal.withInitial { false } + + @Test + fun docInit() { + try { + val config = ConfigurationBuilder() + config.addUrls(ClasspathHelper.forPackage("com.tencent.devops")) + config.setExpandSuperTypes(true) + config.setScanners(Scanners.TypesAnnotated) + val reflections = Reflections(config) + + val doc = document.docInit( + checkMetaData = true, + checkMDData = true, + polymorphism = getAllSubType(reflections), + outputPath = "build/swaggerDoc/", + parametersInfo = getAllApiModelInfo(reflections) + ) + println("${doc.size}|${doc.keys}") + } catch (e: Throwable) { + // 抛错时不影响Test流程 + println("docInit error") + e.printStackTrace() + } + } + + /** + * 获取所有多态类的实现信息 + */ + fun getAllSubType(reflections: Reflections): Map> { + val subTypesClazz = reflections.getTypesAnnotatedWith(JsonSubTypes::class.java) + val res = mutableMapOf>() + subTypesClazz.forEach { + val infoMap = mutableMapOf() + val subTypes = it.getAnnotation(JsonSubTypes::class.java).value +// val typeInfo = it.getAnnotation(JsonTypeInfo::class.java).property + val name = it.getAnnotation(ApiModel::class.java)?.value ?: it.name.split(".").last() + subTypes.forEach { child -> + val childName = child.value.java.getAnnotation(ApiModel::class.java)?.value + ?: child.value.java.name.split(".").last() + infoMap[childName] = child.name + } + res[name] = infoMap + } + return res + } + + fun getAllApiModelInfo(reflections: Reflections): Map> { + val clazz = reflections.getTypesAnnotatedWith(ApiModel::class.java).toList() + val res = mutableMapOf>() + for (i in 0 until clazz.size) { + val it = clazz.getOrNull(i) ?: continue + + println("$i${it.name}") + try { + val name = it.getAnnotation(ApiModel::class.java).value + res[name] = getDataClassParameterDefault(it) + } catch (e: Throwable) { +// println(it.name) +// println(e) + } + } + return res + } + + /** + * 例子: + * ```java + * getDataClassParameterDefault(Class.forName("com.tencent.devops.openapi.pojo.SwaggerDocResponse")) + * ``` + * @param clazz 目标类 + * @return 带默认值的map + */ + @Suppress("ComplexMethod") + fun getDataClassParameterDefault(clazz: Class<*>): Map { + val kClazz = clazz.kotlin + if (!kClazz.isData) return emptyMap() + val constructor = kClazz.constructors.maxByOrNull { it.parameters.size }!! + val parameters = constructor.parameters + val syntheticInit = clazz.declaredConstructors.find { it.modifiers == 4097 } + val argumentsSize = syntheticInit?.parameterTypes?.size ?: parameters.size + val arguments = arrayOfNulls(argumentsSize) + var index = 0 + var offset = 0 + val nullable = mutableMapOf() + parameters.forEach { + if (it.isOptional) { + offset += 1 shl index + } + nullable[it.name ?: ""] = it.type.isMarkedNullable + arguments[index++] = makeStandardArgument(it.type, constructor) + } + for (i in index until argumentsSize - 2) { + arguments[i] = 0 + } + if (syntheticInit != null) { + arguments[argumentsSize - 2] = offset.toInt() + arguments[argumentsSize - 1] = null as DefaultConstructorMarker? + } + val accessor: MemberAccessor = Plugins.getMemberAccessor() + val mock = try { + accessor.newInstance( + syntheticInit ?: constructor.javaConstructor, + { callback: ConstructionDispatcher -> + mockitoConstruction.set(true) + try { + return@newInstance callback.newInstance() + } finally { + mockitoConstruction.set(false) + } + }, + *arguments + ) + } catch (e: Exception) { + throw InstantiationException("Could not instantiate ", e) + } + val res = mutableMapOf() + kClazz.memberProperties.forEach { + // 编译后，属性默认是private,需要设置isAccessible 才可以读取到值 + it.isAccessible = true + res[it.name] = SwaggerDocParameterInfo( + markedNullable = nullable[it.name] ?: false, + defaultValue = checkDefaultValue(it.call(mock).toString()) + ) + } + return res + } + + @Suppress("ComplexCondition") + private fun checkDefaultValue(v: String): String? { + if (v.startsWith("Mock") || v.isBlank() || v == "[]" || v == "{=}" || v == "{}") return null + return v + } + + @Suppress("ComplexMethod") + private fun makeStandardArgument(type: KType, debug: KFunction<*>): Any? { + if (type.isMarkedNullable) return null + return when (type.classifier) { + Boolean::class -> false + Byte::class -> 0.toByte() + Short::class -> 0.toShort() + Char::class -> 0.toChar() + Int::class -> 0 + Long::class -> 0L + Float::class -> 0f + Double::class -> 0.0 + String::class -> "" + Enum::class -> { + null + } + Set::class -> { + type.arguments.firstOrNull()?.let { setOf(makeStandardArgument(it.type!!, debug)) } ?: "" + } + List::class -> { + type.arguments.firstOrNull()?.let { listOf(makeStandardArgument(it.type!!, debug)) } ?: "" + } + ArrayList::class -> { + type.arguments.firstOrNull()?.let { arrayListOf(makeStandardArgument(it.type!!, debug)) } ?: "" + } + Array::class -> { + type.arguments.firstOrNull()?.let { arrayOf(makeStandardArgument(it.type!!, debug)) } ?: "" + } + Map::class -> { + mapOf( + makeStandardArgument( + type.arguments[0].type!!, + debug + ) to makeStandardArgument(type.arguments[1].type!!, debug) + ) + } + else -> { + if (type.javaType is Class<*>) { + Mockito.mock(type.javaType as Class<*>?) + } else { + null + } + } + } + } +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineViewResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineViewResource.kt new file mode 100644 index 00000000000..ffb13854040 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineViewResource.kt @@ -0,0 +1,29 @@ +package com.tencent.devops.process.api.op + +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE +import com.tencent.devops.common.api.pojo.Result +import io.swagger.annotations.Api +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import javax.ws.rs.Consumes +import javax.ws.rs.GET +import javax.ws.rs.HeaderParam +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.core.MediaType + +@Api(tags = ["USER_PIPELINE_VIEW"], description = "用户-流水线视图") +@Path("/op/pipelineViews") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface OpPipelineViewResource { + @ApiOperation("初始化所有动态视图") + @GET + @Path("/initAllView") + fun initAllView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String + ): Result +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceBuildResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceBuildResource.kt index e2bb8914d89..11cbb281add 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceBuildResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceBuildResource.kt @@ -570,6 +570,26 @@ interface ServiceBuildResource { endBeginTime: String? = null ): Result> + @ApiOperation("获取流水线构建历史, 返回buildid") + @GET + @Path("/{projectId}/batch_get_builds") + fun getBuilds( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + @ApiParam("流水线ID", required = false) + @QueryParam("pipelineId") + pipelineId: String?, + @ApiParam("状态id", required = false) + @QueryParam("buildStatus") + buildStatus: Set? = null, + @QueryParam("channelCode") + channelCode: ChannelCode = ChannelCode.BS + ): Result> + @ApiOperation("根据流水线id获取最新执行信息") @POST // @Path("/projects/{projectId}/getPipelineLatestBuild") diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineViewResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineViewResource.kt new file mode 100644 index 00000000000..e822c524d3d --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineViewResource.kt @@ -0,0 +1,201 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.api.service + +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.process.pojo.Pipeline +import com.tencent.devops.process.pojo.PipelineCollation +import com.tencent.devops.process.pojo.PipelineSortType +import com.tencent.devops.process.pojo.classify.PipelineNewView +import com.tencent.devops.process.pojo.classify.PipelineNewViewSummary +import com.tencent.devops.process.pojo.classify.PipelineViewForm +import com.tencent.devops.process.pojo.classify.PipelineViewId +import com.tencent.devops.process.pojo.classify.PipelineViewPipelinePage +import io.swagger.annotations.Api +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import javax.ws.rs.Consumes +import javax.ws.rs.DELETE +import javax.ws.rs.GET +import javax.ws.rs.HeaderParam +import javax.ws.rs.POST +import javax.ws.rs.PUT +import javax.ws.rs.Path +import javax.ws.rs.PathParam +import javax.ws.rs.Produces +import javax.ws.rs.QueryParam +import javax.ws.rs.core.MediaType + +@Api(tags = ["SERVICE_PIPELINE_VIEW"], description = "服务-流水线视图") +@Path("/service/pipelineView/{projectId}") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface ServicePipelineViewResource { + @ApiOperation("用户获取视图流水线编排列表") + @GET + @Path("/listViewPipelines") + fun listViewPipelines( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + @ApiParam("第几页", required = false, defaultValue = "1") + @QueryParam("page") + page: Int?, + @ApiParam("每页多少条", required = false, defaultValue = "20") + @QueryParam("pageSize") + pageSize: Int?, + @ApiParam("流水线排序", required = false, defaultValue = "CREATE_TIME") + @QueryParam("sortType") + sortType: PipelineSortType? = PipelineSortType.CREATE_TIME, + @ApiParam("按流水线过滤", required = false) + @QueryParam("filterByPipelineName") + filterByPipelineName: String?, + @ApiParam("按创建人过滤", required = false) + @QueryParam("filterByCreator") + filterByCreator: String?, + @ApiParam("按标签过滤", required = false) + @QueryParam("filterByLabels") + filterByLabels: String?, + @ApiParam("按视图过滤", required = false) + @QueryParam("filterByViewIds") + filterByViewIds: String? = null, + @ApiParam("用户视图ID,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewId") + viewId: String?, + @ApiParam("用户视图名称,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewName") + viewName: String?, + @ApiParam("维度是否为项目,和viewName搭配使用", required = false) + @QueryParam("isProject") + isProject: Boolean?, + @ApiParam("排序规则", required = false) + @QueryParam("collation") + collation: PipelineCollation?, + @ApiParam("是否展示已删除流水线", required = false) + @QueryParam("showDelete") + showDelete: Boolean? = false + ): Result> + + @ApiOperation("获取视图列表") + @GET + @Path("/list") + fun listView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + @QueryParam("projected") + @ApiParam(value = "是否为项目流水线组 , 为空时不区分", required = false) + projected: Boolean? = null, + @QueryParam("viewType") + @ApiParam(value = "流水线组类型 , 1--动态, 2--静态 , 为空时不区分", required = false) + viewType: Int? = null + ): Result> + + @ApiOperation("添加视图") + @POST + @Path("") + fun addView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + pipelineView: PipelineViewForm + ): Result + + @ApiOperation("获取视图") + @GET + @Path("") + fun getView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String, + @ApiParam("用户视图ID,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewId") + viewId: String?, + @ApiParam("用户视图名称,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewName") + viewName: String?, + @ApiParam("维度是否为项目,和viewName搭配使用", required = false) + @QueryParam("isProject") + isProject: Boolean? + ): Result + + @ApiOperation("删除视图") + @DELETE + @Path("") + fun deleteView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + @ApiParam("用户视图ID,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewId") + viewId: String?, + @ApiParam("用户视图名称,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewName") + viewName: String?, + @ApiParam("维度是否为项目,和viewName搭配使用", required = false) + @QueryParam("isProject") + isProject: Boolean? + ): Result + + @ApiOperation("更改视图") + @PUT + @Path("") + fun updateView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String, + @ApiParam("用户视图ID,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewId") + viewId: String?, + @ApiParam("用户视图名称,表示用户当前所在视图 viewId和viewName 选其一填入", required = false) + @QueryParam("viewName") + viewName: String?, + @ApiParam("维度是否为项目,和viewName搭配使用", required = false) + @QueryParam("isProject") + isProject: Boolean?, + pipelineView: PipelineViewForm + ): Result +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineResource.kt index 5ebdabc5528..1402bec8696 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineResource.kt @@ -32,20 +32,23 @@ import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE import com.tencent.devops.common.api.pojo.Page import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.pipeline.Model -import com.tencent.devops.process.engine.pojo.PipelineInfo import com.tencent.devops.common.pipeline.pojo.MatrixPipelineInfo +import com.tencent.devops.process.engine.pojo.PipelineInfo import com.tencent.devops.process.pojo.Permission import com.tencent.devops.process.pojo.Pipeline +import com.tencent.devops.process.pojo.PipelineCollation import com.tencent.devops.process.pojo.PipelineCopy import com.tencent.devops.process.pojo.PipelineId import com.tencent.devops.process.pojo.PipelineName import com.tencent.devops.process.pojo.PipelineRemoteToken import com.tencent.devops.process.pojo.PipelineSortType -import com.tencent.devops.process.pojo.PipelineStatus import com.tencent.devops.process.pojo.PipelineStageTag +import com.tencent.devops.process.pojo.PipelineStatus import com.tencent.devops.process.pojo.app.PipelinePage import com.tencent.devops.process.pojo.classify.PipelineViewAndPipelines import com.tencent.devops.process.pojo.classify.PipelineViewPipelinePage +import com.tencent.devops.process.pojo.pipeline.BatchDeletePipeline +import com.tencent.devops.process.pojo.pipeline.PipelineCount import com.tencent.devops.process.pojo.setting.PipelineModelAndSetting import com.tencent.devops.process.pojo.setting.PipelineSetting import io.swagger.annotations.Api @@ -288,7 +291,6 @@ interface UserPipelineResource { @ApiOperation("删除流水线编排") @DELETE - // @Path("/projects/{projectId}/pipelines/{pipelineId}/") @Path("/{projectId}/{pipelineId}/") fun softDelete( @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -302,6 +304,16 @@ interface UserPipelineResource { pipelineId: String ): Result + @ApiOperation("批量删除流水线编排") + @DELETE + @Path("/batchDelete") + fun batchDelete( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + batchDeletePipeline: BatchDeletePipeline + ): Result> + @ApiOperation("删除流水线版本") @DELETE @Path("/{projectId}/{pipelineId}/{version}/") @@ -367,9 +379,18 @@ interface UserPipelineResource { @ApiParam("按标签过滤", required = false) @QueryParam("filterByLabels") filterByLabels: String?, - @ApiParam("用户视图ID", required = true) + @ApiParam("按视图过滤", required = false) + @QueryParam("filterByViewIds") + filterByViewIds: String? = null, + @ApiParam("用户视图ID,表示用户当前所在视图", required = true) @QueryParam("viewId") - viewId: String + viewId: String, + @ApiParam("排序规则", required = false) + @QueryParam("collation") + collation: PipelineCollation?, + @ApiParam("是否展示已删除流水线", required = false) + @QueryParam("showDelete") + showDelete: Boolean? = false ): Result> @ApiOperation("有权限流水线编排列表") @@ -445,7 +466,6 @@ interface UserPipelineResource { @ApiOperation("列出等还原回收的流水线列表") @GET - // @Path("/projects/{projectId}/pipelineRecycleList") @Path("/{projectId}/pipelineRecycleList") fun recycleList( @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @@ -462,7 +482,10 @@ interface UserPipelineResource { pageSize: Int?, @ApiParam("流水线排序", required = false, defaultValue = "CREATE_TIME") @QueryParam("sortType") - sortType: PipelineSortType? = PipelineSortType.CREATE_TIME + sortType: PipelineSortType? = PipelineSortType.CREATE_TIME, + @ApiParam("排序规则", required = false) + @QueryParam("collation") + collation: PipelineCollation? ): Result> @ApiOperation("流水线重命名") @@ -559,4 +582,16 @@ interface UserPipelineResource { @ApiParam("yaml内容", required = true) yaml: MatrixPipelineInfo ): Result + + @ApiOperation("获取列表页列表相关的数目") + @GET + @Path("/projects/{projectId}/getCount") + fun getCount( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String + ): Result } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineViewResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineViewResource.kt index 40f54c3d5c2..8c369446460 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineViewResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineViewResource.kt @@ -31,11 +31,18 @@ import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.process.pojo.classify.PipelineNewView -import com.tencent.devops.process.pojo.classify.PipelineNewViewCreate import com.tencent.devops.process.pojo.classify.PipelineNewViewSummary -import com.tencent.devops.process.pojo.classify.PipelineNewViewUpdate +import com.tencent.devops.process.pojo.classify.PipelineViewBulkAdd +import com.tencent.devops.process.pojo.classify.PipelineViewBulkRemove +import com.tencent.devops.process.pojo.classify.PipelineViewDict +import com.tencent.devops.process.pojo.classify.PipelineViewForm +import com.tencent.devops.process.pojo.classify.PipelineViewHitFilters import com.tencent.devops.process.pojo.classify.PipelineViewId +import com.tencent.devops.process.pojo.classify.PipelineViewMatchDynamic +import com.tencent.devops.process.pojo.classify.PipelineViewPipelineCount +import com.tencent.devops.process.pojo.classify.PipelineViewPreview import com.tencent.devops.process.pojo.classify.PipelineViewSettings +import com.tencent.devops.process.pojo.classify.PipelineViewTopForm import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam @@ -48,6 +55,7 @@ import javax.ws.rs.PUT import javax.ws.rs.Path import javax.ws.rs.PathParam import javax.ws.rs.Produces +import javax.ws.rs.QueryParam import javax.ws.rs.core.MediaType @Api(tags = ["USER_PIPELINE_VIEW"], description = "用户-流水线视图") @@ -58,7 +66,6 @@ interface UserPipelineViewResource { @ApiOperation("获取视图设置") @GET @Path("/projects/{projectId}/settings") - // @Path("/{projectId}/settings") fun getViewSettings( @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) @@ -71,7 +78,6 @@ interface UserPipelineViewResource { @ApiOperation("更新视图设置") @POST @Path("/projects/{projectId}/settings") - // @Path("/{projectId}/settings") fun updateViewSettings( @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) @@ -85,7 +91,6 @@ interface UserPipelineViewResource { @ApiOperation("获取所有视图") @GET @Path("/projects/{projectId}/") - // @Path("/{projectId}/") fun getViews( @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) @@ -95,10 +100,27 @@ interface UserPipelineViewResource { projectId: String ): Result> + @ApiOperation("获取视图列表") + @GET + @Path("/projects/{projectId}/list") + fun listView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + @QueryParam("projected") + @ApiParam(value = "是否为项目流水线组 , 为空时不区分", required = false) + projected: Boolean? = null, + @QueryParam("viewType") + @ApiParam(value = "流水线组类型 , 1--动态, 2--静态 , 为空时不区分", required = false) + viewType: Int? = null + ): Result> + @ApiOperation("添加视图") @POST @Path("/projects/{projectId}/") - // @Path("/{projectId}/") fun addView( @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) @@ -106,13 +128,12 @@ interface UserPipelineViewResource { @ApiParam("项目ID", required = true) @PathParam("projectId") projectId: String, - pipelineView: PipelineNewViewCreate + pipelineView: PipelineViewForm ): Result @ApiOperation("获取视图") @GET @Path("/projects/{projectId}/views/{viewId}") - // @Path("/{projectId}/{viewId}") fun getView( @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) @@ -127,7 +148,6 @@ interface UserPipelineViewResource { @ApiOperation("删除视图") @DELETE @Path("/projects/{projectId}/views/{viewId}") - // Path("/{projectId}/{viewId}") fun deleteView( @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) @@ -143,7 +163,6 @@ interface UserPipelineViewResource { @ApiOperation("更改视图") @PUT @Path("/projects/{projectId}/views/{viewId}") - // @Path("/{projectId}/{viewId}") fun updateView( @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) @HeaderParam(AUTH_HEADER_USER_ID) @@ -153,6 +172,123 @@ interface UserPipelineViewResource { @ApiParam("标签ID", required = true) @PathParam("viewId") viewId: String, - pipelineView: PipelineNewViewUpdate + pipelineView: PipelineViewForm + ): Result + + @ApiOperation("置顶视图") + @POST + @Path("/projects/{projectId}/views/{viewId}/top") + fun topView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String, + @ApiParam("标签ID", required = true) + @PathParam("viewId") + viewId: String, + pipelineViewTopForm: PipelineViewTopForm + ): Result + + @ApiOperation("预览视图") + @POST + @Path("/projects/{projectId}/preview") + fun preview( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String, + pipelineView: PipelineViewForm + ): Result + + @ApiOperation("获取流水线组与流水线的对应关系") + @GET + @Path("/projects/{projectId}/dict") + fun dict( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String + ): Result + + @ApiOperation("流水线组过滤条件") + @GET + @Path("/projects/{projectId}/getHitFilters") + fun getHitFilters( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String, + @QueryParam("pipelineId") + pipelineId: String, + @QueryParam("viewId") + viewId: String + ): Result + + @ApiOperation("命中动态组情况") + @POST + @Path("/projects/{projectId}/matchDynamicView") + fun matchDynamicView( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String, + pipelineViewMatchDynamic: PipelineViewMatchDynamic + ): Result> + + @ApiOperation("批量添加") + @POST + @Path("/projects/{projectId}/bulkAdd") + fun bulkAdd( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String, + bulkAdd: PipelineViewBulkAdd + ): Result + + @ApiOperation("批量移除") + @POST + @Path("/projects/{projectId}/bulkRemove") + fun bulkRemove( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String, + bulkRemove: PipelineViewBulkRemove ): Result + + @ApiOperation("根据流水线ID获取视图(流水线组)") + @GET + @Path("/projects/{projectId}/pipelines/{pipelineId}/listViews") + fun listViewByPipelineId( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @PathParam("projectId") + projectId: String, + @PathParam("pipelineId") + pipelineId: String + ): Result> + + @ApiOperation("根据视图ID获取当前流水线的具体数目") + @GET + @Path("/projects/{projectId}/views/{viewId}/pipelineCount") + fun pipelineCount( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + @ApiParam("标签ID", required = true) + @PathParam("viewId") + viewId: String + ): Result } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserSubPipelineInfoResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserSubPipelineInfoResource.kt new file mode 100644 index 00000000000..7d911930b23 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserSubPipelineInfoResource.kt @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.api.user + +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.process.pojo.pipeline.SubPipelineStartUpInfo +import io.swagger.annotations.Api +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import javax.ws.rs.Consumes +import javax.ws.rs.GET +import javax.ws.rs.HeaderParam +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.QueryParam +import javax.ws.rs.core.MediaType + +@Api(tags = ["USER_SUBPIPELINE"], description = "获取子流水线信息") +@Path("/user/subpipeline") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface UserSubPipelineInfoResource { + @ApiOperation("获取子流水线启动参数") + @GET + @Path("/manualStartupInfo") + fun subpipManualStartupInfo( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @QueryParam("projectId") + projectId: String, + @ApiParam("流水线ID", required = false, defaultValue = "") + @QueryParam("subPip") + pipelineId: String + ): Result> +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/PipelineViewType.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/PipelineViewType.kt new file mode 100644 index 00000000000..09d1e077d55 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/PipelineViewType.kt @@ -0,0 +1,7 @@ +package com.tencent.devops.process.constant + +object PipelineViewType { + const val UNCLASSIFIED = -1 + const val DYNAMIC = 1 + const val STATIC = 2 +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt index ad67f3e0e67..4d31db3096d 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt @@ -149,6 +149,7 @@ object ProcessMessageCode { const val ERROR_PARUS_PIEPLINE_IS_RUNNINT = "2101905" // 暂停的流水线已开始运行 const val ERROR_ELEMENT_TOO_LONG = "2101906" // {0} element大小越界 const val ERROR_JOB_RUNNING = "2101907" // job非完成态，不能进行重试 + const val ERROR_RETRY_STAGE_NOT_FAILED = "2101911" // stage非失败状态，不能进行重试 const val ERROR_NO_BUILD_EXISTS_BY_ID = "2101100" // 流水线构建[{0}]不存在 const val ERROR_NO_PIPELINE_EXISTS_BY_ID = "2101101" // 流水线[{0}]不存在 @@ -190,9 +191,11 @@ object ProcessMessageCode { const val ERROR_BUILD_TASK_ACROSS_PROJECT_PARAM_TARGETPROJECTID = "2101123" const val ERROR_BUILD_TASK_QUALITY_IN = "2101137" // 质量红线(准入)检测失败 + // 质量红线(准入)配置有误：Fail to find quality gate intercept element const val ERROR_BUILD_TASK_QUALITY_IN_INTERCEPT = "2101908" const val ERROR_BUILD_TASK_QUALITY_OUT = "2101909" // 质量红线(准出)检测失败 + // 质量红线(准出)配置有误：Fail to find quality gate intercept element const val ERROR_BUILD_TASK_QUALITY_OUT_INTERCEPT = "2101910" @@ -243,6 +246,7 @@ object ProcessMessageCode { // 其他构建进程挂掉的参考信息，自由添加方便打印卫通日志里 const val BUILD_WORKER_DEAD_ERROR = "2101318" + // 构建机Agent详情链接 const val BUILD_AGENT_DETAIL_LINK_ERROR = "2101319" @@ -253,4 +257,11 @@ object ProcessMessageCode { const val ERROR_GROUP_COUNT_EXCEEDS_LIMIT = "2101401" // 一个项目标签组不能超过10个 const val ERROR_LABEL_COUNT_EXCEEDS_LIMIT = "2101402" // 同一分组下最多可添加12个标签 const val ERROR_LABEL_NAME_TOO_LONG = "2101403" // 一个标签最多输入20个字符 + + // 流水线组错误21016开头 + const val ERROR_VIEW_GROUP_NO_PERMISSION = "2101601" // 没有修改流水线组权限 + const val ERROR_VIEW_GROUP_IS_PROJECT_NO_SAME = "2101602" // 流水线组的视图范围不一致 + const val ERROR_VIEW_EXCEED_THE_LIMIT = "2101603" // 流水线组创建太多了 + const val ERROR_VIEW_DUPLICATE_NAME = "2101604" // 流水线组名称重复 + const val ERROR_VIEW_NAME_ILLEGAL = "2101605" // 流水线组名称不合法 } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineInfo.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineInfo.kt index a385ffbdfdd..10556b02d9f 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineInfo.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineInfo.kt @@ -28,22 +28,43 @@ package com.tencent.devops.process.engine.pojo import com.tencent.devops.common.pipeline.enums.ChannelCode +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty +@ApiModel("流水线信息") data class PipelineInfo( + @ApiModelProperty("项目ID") val projectId: String, + @ApiModelProperty("流水线DI") val pipelineId: String, + @ApiModelProperty("模板ID") val templateId: String?, + @ApiModelProperty("流水线名称") val pipelineName: String, + @ApiModelProperty("流水线描述") val pipelineDesc: String, + @ApiModelProperty("版本") var version: Int = 1, + @ApiModelProperty("创建时间") val createTime: Long = 0, + @ApiModelProperty("更新时间") val updateTime: Long = 0, + @ApiModelProperty("创建者") val creator: String, + @ApiModelProperty("上一次的更新者") val lastModifyUser: String, + @ApiModelProperty("渠道号") val channelCode: ChannelCode, + @ApiModelProperty("是否能够手动启动") val canManualStartup: Boolean, + @ApiModelProperty("是否可以跳过") val canElementSkip: Boolean, + @ApiModelProperty("任务数") val taskCount: Int, + @ApiModelProperty("版本名称") var versionName: String = "init", - val id: Long? + @ApiModelProperty("ID") + val id: Long?, + @ApiModelProperty("流水线组名称列表", required = false) + var viewNames: List? = null ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildHistory.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildHistory.kt index da97ed8aec7..8c314cd504b 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildHistory.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildHistory.kt @@ -46,9 +46,9 @@ data class BuildHistory( val buildNum: Int?, @ApiModelProperty("编排文件版本号", required = true) val pipelineVersion: Int, - @ApiModelProperty("开始时间", required = true) + @ApiModelProperty("流水线的执行开始时间", required = true) val startTime: Long, - @ApiModelProperty("结束时间", required = true) + @ApiModelProperty("流水线的执行结束时间", required = true) val endTime: Long?, @ApiModelProperty("状态", required = true) val status: String, @@ -88,7 +88,7 @@ data class BuildHistory( var buildMsg: String?, @ApiModelProperty("自定义构建版本号", required = false) val buildNumAlias: String? = null, - @ApiModelProperty("更新时间", required = false) + @ApiModelProperty("流水线编排的最后更新时间", required = false) val updateTime: Long? = null, @ApiModelProperty("并发时,设定的group", required = false) var concurrencyGroup: String? = null diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildHistoryWithVars.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildHistoryWithVars.kt index 40baf29625d..4119d844ca2 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildHistoryWithVars.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildHistoryWithVars.kt @@ -62,6 +62,8 @@ data class BuildHistoryWithVars( val material: List?, @ApiModelProperty("排队于", required = false) val queueTime: Long?, + @ApiModelProperty("排队位置", required = false) + val currentQueuePosition: Int = 0, @ApiModelProperty("构件列表", required = false) val artifactList: List?, @ApiModelProperty("备注", required = false) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/Pipeline.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/Pipeline.kt index 8a4b646c60b..69f5902a1c3 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/Pipeline.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/Pipeline.kt @@ -102,5 +102,25 @@ data class Pipeline( @ApiModelProperty("自定义构建号规则", required = false) var buildNumRule: String? = null, @ApiModelProperty("编排详情", required = false) - var model: Model? = null + var model: Model? = null, + @ApiModelProperty("流水线组名称列表", required = false) + var viewNames: List? = null, + @ApiModelProperty("最后一次构建的构建信息", required = false) + var lastBuildMsg: String? = null, + @ApiModelProperty("最后一次构建所有的任务个数", required = false) + var lastBuildTotalCount: Int? = null, + @ApiModelProperty("最后一次构建已完成的任务个数", required = false) + var lastBuildFinishCount: Int? = null, + @ApiModelProperty("触发方式", required = false) + var trigger: String? = null, + @ApiModelProperty("webhook仓库别名", required = false) + var webhookAliasName: String? = null, + @ApiModelProperty("webhook提交信息", required = false) + var webhookMessage: String? = null, + @ApiModelProperty("webhook仓库地址", required = false) + var webhookRepoUrl: String? = null, + @ApiModelProperty("webhook类型", required = false) + var webhookType: String? = null, + @ApiModelProperty("是否已删除", required = false) + var delete: Boolean? = false ) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/tstack/TStackDispatchType.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineCollation.kt similarity index 73% rename from src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/tstack/TStackDispatchType.kt rename to src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineCollation.kt index 8893635124c..3fecd7a32ec 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/tstack/TStackDispatchType.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineCollation.kt @@ -25,16 +25,17 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.common.pipeline.type.tstack +package com.tencent.devops.process.pojo -import com.fasterxml.jackson.annotation.JsonProperty -import com.tencent.devops.common.pipeline.type.BuildType -import com.tencent.devops.common.pipeline.type.DispatchType +/** + * 流水线排序规则 + */ +enum class PipelineCollation { + DEFAULT, -data class TStackDispatchType(@JsonProperty("value") val tstackAgentId: String) : DispatchType(tstackAgentId) { - override fun cleanDataBeforeSave() = Unit + ASC, - override fun replaceField(variables: Map) = Unit + DESC, - override fun buildType() = BuildType.valueOf(BuildType.TSTACK.name) + ; } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineCopy.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineCopy.kt index 6e1d7f1e266..ac9f431f04d 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineCopy.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineCopy.kt @@ -36,8 +36,8 @@ data class PipelineCopy( val name: String, @ApiModelProperty("描述", required = false) val desc: String?, - @ApiModelProperty("分组名称", required = false) - val group: String = "", - @ApiModelProperty("是否收藏", required = false) - val hasCollect: Boolean? = false + @ApiModelProperty("标签", required = false) + var labels: List = emptyList(), + @ApiModelProperty("静态流水线组", required = false) + var staticViews: List = emptyList() ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineStatus.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineStatus.kt index e9aaacfcd68..288285ccbfc 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineStatus.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineStatus.kt @@ -60,5 +60,9 @@ data class PipelineStatus( @ApiModelProperty("当前运行的构建的个数", required = true) val runningBuildCount: Int, @ApiModelProperty("是否被收藏", required = true) - val hasCollect: Boolean + val hasCollect: Boolean, + @ApiModelProperty("最后一次构建所有的任务个数", required = false) + var lastBuildTotalCount: Int? = null, + @ApiModelProperty("最后一次构建已完成的任务个数", required = false) + var lastBuildFinishCount: Int? = null ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewView.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewView.kt index 02107d9ce6d..cc11a70f585 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewView.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewView.kt @@ -50,5 +50,9 @@ data class PipelineNewView( @ApiModelProperty("逻辑符", required = false) val logic: Logic, @ApiModelProperty("流水线视图过滤器列表", required = false) - val filters: List + val filters: List, + @ApiModelProperty("视图类型", required = true) + val viewType: Int, + @ApiModelProperty("流水线ID列表", required = true) + val pipelineIds: List ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewViewSummary.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewViewSummary.kt index dc1325a1788..ab3adae48a0 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewViewSummary.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineNewViewSummary.kt @@ -45,5 +45,11 @@ data class PipelineNewViewSummary( @ApiModelProperty("更新时间", required = false) val updateTime: Long, @ApiModelProperty("创建者", required = false) - val creator: String + val creator: String, + @ApiModelProperty("是否置顶", required = false) + val top: Boolean = false, + @ApiModelProperty("流水线组类型,1--动态,2--静态", required = true) + val viewType: Int, + @ApiModelProperty("流水线个数", required = true) + val pipelineCount: Int ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewBulkAdd.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewBulkAdd.kt new file mode 100644 index 00000000000..9ed5b9fc4d5 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewBulkAdd.kt @@ -0,0 +1,12 @@ +package com.tencent.devops.process.pojo.classify + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("流水线组批量添加") +data class PipelineViewBulkAdd( + @ApiModelProperty("流水线ID列表") + val pipelineIds: List, + @ApiModelProperty("视图ID列表") + val viewIds: List +) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewBulkRemove.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewBulkRemove.kt new file mode 100644 index 00000000000..b94165f17cc --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewBulkRemove.kt @@ -0,0 +1,12 @@ +package com.tencent.devops.process.pojo.classify + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("流水线组批量移除") +data class PipelineViewBulkRemove( + @ApiModelProperty("流水线ID列表") + val pipelineIds: List, + @ApiModelProperty("视图ID") + val viewId: String +) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewDict.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewDict.kt new file mode 100644 index 00000000000..f2eaa4dc7e1 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewDict.kt @@ -0,0 +1,38 @@ +package com.tencent.devops.process.pojo.classify + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("流水线组与流水线的对应关系") +data class PipelineViewDict( + @ApiModelProperty("个人流水线组列表") + val personalViewList: List, + @ApiModelProperty("项目流水线列表") + val projectViewList: List +) { + @ApiModel("流水线组信息") + data class ViewInfo( + @ApiModelProperty("流水线组ID") + val viewId: String, + @ApiModelProperty("流水线组名") + val viewName: String, + @ApiModelProperty("流水线列表") + val pipelineList: List + ) { + @ApiModel("流水线信息") + data class PipelineInfo( + @ApiModelProperty("流水线ID") + val pipelineId: String, + @ApiModelProperty("流水线名称") + val pipelineName: String, + @ApiModelProperty("流水线组ID") + val viewId: String, + @ApiModelProperty("是否删除") + val delete: Boolean + ) + } + + companion object { + val EMPTY = PipelineViewDict(emptyList(), emptyList()) + } +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewForm.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewForm.kt new file mode 100644 index 00000000000..712308e2b66 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewForm.kt @@ -0,0 +1,24 @@ +package com.tencent.devops.process.pojo.classify + +import com.tencent.devops.process.constant.PipelineViewType +import com.tencent.devops.process.pojo.classify.enums.Logic +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("流水线视图表单") +data class PipelineViewForm( + @ApiModelProperty("ID", required = false) + val id: String? = null, + @ApiModelProperty("视图名称", required = false) + var name: String, + @ApiModelProperty("是否项目", required = false) + val projected: Boolean, + @ApiModelProperty("流水线组类型,1--动态,2--静态") + var viewType: Int = PipelineViewType.UNCLASSIFIED, + @ApiModelProperty("逻辑符", required = false) + val logic: Logic = Logic.AND, + @ApiModelProperty("流水线视图过滤器列表", required = false) + val filters: List = emptyList(), + @ApiModelProperty("流水线列表", required = false) + val pipelineIds: List? = null +) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewHitFilters.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewHitFilters.kt new file mode 100644 index 00000000000..6a3ae7c8cba --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewHitFilters.kt @@ -0,0 +1,30 @@ +package com.tencent.devops.process.pojo.classify + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("流水线组命中情况") +data class PipelineViewHitFilters( + @ApiModelProperty("条件列表") + val filters: MutableList, + @ApiModelProperty("条件关系") + val logic: String +) { + data class FilterInfo( + @ApiModelProperty("关键字") + val key: String, + @ApiModelProperty("命中列表") + val hits: MutableList + ) { + data class Hit( + @ApiModelProperty("是否命中") + val hit: Boolean, + @ApiModelProperty("对应的值") + val value: String + ) + } + + companion object { + val EMPTY = PipelineViewHitFilters(mutableListOf(), "") + } +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewMatchDynamic.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewMatchDynamic.kt new file mode 100644 index 00000000000..60033073de2 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewMatchDynamic.kt @@ -0,0 +1,20 @@ +package com.tencent.devops.process.pojo.classify + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("命中动态组情况") +data class PipelineViewMatchDynamic( + @ApiModelProperty("流水线名称") + val pipelineName: String, + @ApiModelProperty("标签列表") + val labels: List +) { + @ApiModel("标签信息") + data class LabelInfo( + @ApiModelProperty("标签分组id", required = false) + val groupId: String, + @ApiModelProperty("标签id列表", required = false) + val labelIds: List + ) +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewPipelineCount.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewPipelineCount.kt new file mode 100644 index 00000000000..c99abf3c62d --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewPipelineCount.kt @@ -0,0 +1,16 @@ +package com.tencent.devops.process.pojo.classify + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("流水线组--详细数目") +data class PipelineViewPipelineCount( + @ApiModelProperty("可查看流水线数目") + val normalCount: Int, + @ApiModelProperty("已删除流水线数目") + val deleteCount: Int +) { + companion object { + val DEFAULT = PipelineViewPipelineCount(0, 0) + } +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewPipelinePage.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewPipelinePage.kt index 66e23390636..829edd571cf 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewPipelinePage.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewPipelinePage.kt @@ -49,12 +49,11 @@ data class PipelineViewPipelinePage( pageSize: Int, count: Long, records: List - ) : - this( - count = count, - page = page, - pageSize = pageSize, - totalPages = if (pageSize == -1) 1 else ceil(count * 1.0 / pageSize).toInt(), - records = records - ) + ) : this( + count = count, + page = page, + pageSize = pageSize, + totalPages = if (pageSize == -1) 1 else ceil(count * 1.0 / pageSize).toInt(), + records = records + ) } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewPreview.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewPreview.kt new file mode 100644 index 00000000000..8d19a924b33 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewPreview.kt @@ -0,0 +1,27 @@ +package com.tencent.devops.process.pojo.classify + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("Pipeline视图预览") +data class PipelineViewPreview( + @ApiModelProperty("新增的流水线ID列表", required = true) + val addedPipelineInfos: List, + @ApiModelProperty("删除的流水线ID列表", required = true) + val removedPipelineInfos: List, + @ApiModelProperty("保留的流水线ID列表", required = true) + val reservePipelineInfos: List +) { + data class PipelineInfo( + @ApiModelProperty("名称", required = true) + val pipelineName: String, + @ApiModelProperty("ID", required = true) + val pipelineId: String, + @ApiModelProperty("是否删除", required = true) + val delete: Boolean + ) + + companion object { + val EMPTY = PipelineViewPreview(emptyList(), emptyList(), emptyList()) + } +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewTopForm.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewTopForm.kt new file mode 100644 index 00000000000..97d79d6f6a7 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/PipelineViewTopForm.kt @@ -0,0 +1,10 @@ +package com.tencent.devops.process.pojo.classify + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("流水线视图表单") +data class PipelineViewTopForm( + @ApiModelProperty("是否生效", required = true) + val enabled: Boolean +) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/enums/Logic.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/enums/Logic.kt index ba94590596c..61488b6465b 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/enums/Logic.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/classify/enums/Logic.kt @@ -29,5 +29,15 @@ package com.tencent.devops.process.pojo.classify.enums enum class Logic { AND, - OR + OR; + + companion object { + fun of(value: String): Logic { + return try { + valueOf(value) + } catch (e: Exception) { + return AND + } + } + } } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/code/WebhookInfo.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/code/WebhookInfo.kt index 1001a862fa2..6251bc002a3 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/code/WebhookInfo.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/code/WebhookInfo.kt @@ -34,6 +34,8 @@ data class WebhookInfo( val webhookRepoUrl: String?, @ApiModelProperty("分支名", required = false) val webhookBranch: String?, + @ApiModelProperty("别名", required = false) + val webhookAliasName: String?, @ApiModelProperty("webhook类型", required = false) val webhookType: String?, @ApiModelProperty("事件类型", required = false) @@ -55,5 +57,13 @@ data class WebhookInfo( // mr url val mrUrl: String?, // webhook仓库授权用户 - val repoAuthUser: String? + val repoAuthUser: String?, + // tag 名称 + val tagName: String?, + // issue iid, + val issueIid: String?, + // note id + val noteId: String?, + // review id + val reviewId: String? ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/BatchDeletePipeline.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/BatchDeletePipeline.kt new file mode 100644 index 00000000000..0b6338126bc --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/BatchDeletePipeline.kt @@ -0,0 +1,12 @@ +package com.tencent.devops.process.pojo.pipeline + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("批量删除流水线") +data class BatchDeletePipeline( + @ApiModelProperty("项目ID") + val projectId: String, + @ApiModelProperty("流水线ID列表") + val pipelineIds: List +) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/ModelDetail.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/ModelDetail.kt index d1be8d89851..0b2d622c57a 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/ModelDetail.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/ModelDetail.kt @@ -49,7 +49,7 @@ data class ModelDetail( val startTime: Long, @ApiModelProperty("End time", required = false) val endTime: Long?, - @ApiModelProperty("Build status", required = true) + @ApiModelProperty(value = "Build status", required = true) val status: String, @ApiModelProperty("Build Model", required = true) val model: Model, diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/PipelineCount.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/PipelineCount.kt new file mode 100644 index 00000000000..8fe47b9503e --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/PipelineCount.kt @@ -0,0 +1,18 @@ +package com.tencent.devops.process.pojo.pipeline + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("流水线数量相关") +data class PipelineCount( + @ApiModelProperty("全部流水线个数", required = true) + var totalCount: Int, + @ApiModelProperty("我的收藏个数", required = true) + var myFavoriteCount: Int, + @ApiModelProperty("我的流水线的个数", required = true) + var myPipelineCount: Int, + @ApiModelProperty("回收站流水线的个数", required = true) + var recycleCount: Int, + @ApiModelProperty("最近使用的流水线的个数", required = true) + val recentUseCount: Int +) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/template/TemplateInstanceCreate.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/template/TemplateInstanceCreate.kt index 4c873bccbae..14a29f6932f 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/template/TemplateInstanceCreate.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/template/TemplateInstanceCreate.kt @@ -40,7 +40,7 @@ import io.swagger.annotations.ApiModelProperty data class TemplateInstanceCreate( @ApiModelProperty("流水线名称", required = false) val pipelineName: String, - @ApiModelProperty("构建号", required = false) + @ApiModelProperty("构建号（推荐版本号）", required = false) val buildNo: BuildNo?, @ApiModelProperty("流水线变量列表", required = false) val param: List? = null diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt index 599a53dfb33..41e72e4b55b 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt @@ -79,8 +79,11 @@ const val BK_CI_BUILD_FAIL_TASKS = "BK_CI_BUILD_FAIL_TASKS" const val BK_CI_BUILD_FAIL_TASKNAMES = "BK_CI_BUILD_FAIL_TASKNAMES" const val PIPELINE_VIEW_MY_PIPELINES = "myPipeline" +const val PIPELINE_VIEW_MY_LIST_PIPELINES = "myListPipeline" // 兼容APP , 后面需要干掉 const val PIPELINE_VIEW_FAVORITE_PIPELINES = "collect" const val PIPELINE_VIEW_ALL_PIPELINES = "allPipeline" +const val PIPELINE_VIEW_UNCLASSIFIED = "unclassified" +const val PIPELINE_VIEW_RECENT_USE = "recentUse" const val PIPELINE_MATERIAL_URL = "BK_CI_PIEPLEINE_MATERIAL_URL" // pipeline.material.url const val PIPELINE_MATERIAL_BRANCHNAME = "BK_CI_PIPELINE_MATERIAL_BRANCHNAME" // pipeline.material.branchName @@ -114,10 +117,12 @@ const val PIPELINE_ATOM_TIMEOUT = "BK_CI_ATOM_TIMEOUT" // "流水线插件超时 * 流水线设置-最大排队数量-默认值 */ const val PIPELINE_SETTING_MAX_QUEUE_SIZE_DEFAULT = 10 + /** * 流水线插件设置-失败重试最大值 */ const val TASK_FAIL_RETRY_MAX_COUNT = 5 + /** * 流水线插件设置-失败重试最小值 */ @@ -127,14 +132,17 @@ const val TASK_FAIL_RETRY_MIN_COUNT = 1 * 流水线设置-最大排队数量-最小值 */ const val PIPELINE_SETTING_MAX_QUEUE_SIZE_MIN = 0 + /** * 流水线设置-最大排队数量-最大值 */ const val PIPELINE_SETTING_MAX_QUEUE_SIZE_MAX = 20 + /** * 流水线设置-最大并发数量-默认值 */ const val PIPELINE_SETTING_MAX_CON_QUEUE_SIZE_DEFAULT = 50 + /** * 流水线设置-最大并发数量-最大值 */ @@ -164,18 +172,22 @@ const val PIPELINE_TASK_MESSAGE_STRING_LENGTH_MAX = 4000 * 流水线设置-流水线错误信息入库长度最大值 单位:分钟 */ const val PIPELINE_MESSAGE_STRING_LENGTH_MAX = 30000 + /** * 流水线设置-流水线最大任务并发数量-最大值 */ const val PIPELINE_CON_RUNNING_CONTAINER_SIZE_MAX = 30 + /** * 流水线设置-矩阵内最大并发数量-默认值 */ const val PIPELINE_MATRIX_MAX_CON_RUNNING_SIZE_DEFAULT = 5 + /** * 流水线设置-矩阵内最大并发数量-最大值 */ const val PIPELINE_MATRIX_CON_RUNNING_SIZE_MAX = 20 + /** * 流水线设置-Stage内最大分裂后Job数量-最大值 */ @@ -191,10 +203,12 @@ const val PIPELINE_TIME_START = "BK_CI_BUILD_START_TIME" // "pipeline.time.start const val PIPELINE_TIME_END = "BK_CI_BUILD_END_TIME" // "pipeline.time.end" const val PIPELINE_BUILD_MSG = "BK_CI_BUILD_MSG" + /** * 流水线设置-CONCURRENCY GROUP 并发组-默认值 */ const val PIPELINE_SETTING_CONCURRENCY_GROUP_DEFAULT = "\${{ci.pipeline_id}}" + /** * 保存流水线编排的最大个数 */ diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/PipelineVarUtil.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/PipelineVarUtil.kt index 1bb267733df..d4ba2ad3ca1 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/PipelineVarUtil.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/PipelineVarUtil.kt @@ -65,6 +65,7 @@ import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_ISSUE_MIL import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_ISSUE_OWNER import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_ISSUE_STATE import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_ISSUE_TITLE +import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_MR_REVIEWERS import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_REVIEW_ID import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_REVIEW_IID import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_REVIEW_OWNER @@ -220,7 +221,8 @@ object PipelineVarUtil { "ci.note_comment" to PIPELINE_WEBHOOK_NOTE_COMMENT, "ci.note_id" to PIPELINE_WEBHOOK_NOTE_ID, "ci.action" to PIPELINE_GIT_ACTION, - "ci.build_url" to PIPELINE_BUILD_URL + "ci.build_url" to PIPELINE_BUILD_URL, + "ci.mr_reviewers" to BK_REPO_GIT_WEBHOOK_MR_REVIEWERS ) /** diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineFavorDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineFavorDao.kt index 49b3c9a75d5..61d014d3022 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineFavorDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineFavorDao.kt @@ -116,6 +116,15 @@ class PipelineFavorDao { } } + fun countByUserId(dslContext: DSLContext, projectId: String, userId: String): Int { + with(TPipelineFavor.T_PIPELINE_FAVOR) { + return dslContext.selectCount().from(this) + .where(CREATE_USER.eq(userId)) + .and(PROJECT_ID.eq(projectId)) + .fetchOne()?.value1() ?: 0 + } + } + fun listByPipelineId( dslContext: DSLContext, userId: String, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineRecentUseDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineRecentUseDao.kt new file mode 100644 index 00000000000..1c636c71fd9 --- /dev/null +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineRecentUseDao.kt @@ -0,0 +1,58 @@ +package com.tencent.devops.process.dao + +import com.tencent.devops.model.process.tables.TPipelineRecentUse +import org.jooq.DSLContext +import org.springframework.stereotype.Repository +import java.time.LocalDateTime + +/** + * 最近使用的流水线 + */ +@Repository +class PipelineRecentUseDao { + /** + * 新增记录 + */ + fun add( + dslContext: DSLContext, + projectId: String, + userId: String, + pipelineId: String + ) { + val now = LocalDateTime.now() + with(TPipelineRecentUse.T_PIPELINE_RECENT_USE) { + dslContext.insertInto( + this, + PROJECT_ID, + USER_ID, + PIPELINE_ID, + USE_TIME + ).values( + projectId, + userId, + pipelineId, + now + ).onDuplicateKeyUpdate().set(USE_TIME, now).execute() + } + } + + /** + * 流水线列表 + */ + fun listRecentPipelineIds( + dslContext: DSLContext, + projectId: String, + userId: String, + limit: Int + ): List { + return with(TPipelineRecentUse.T_PIPELINE_RECENT_USE) { + dslContext.select(PIPELINE_ID) + .from(this) + .where(PROJECT_ID.eq(projectId)) + .and(USER_ID.eq(userId)) + .orderBy(USE_TIME.desc()) + .limit(limit) + .fetch(0, String::class.java) + } + } +} diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/label/PipelineViewDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/label/PipelineViewDao.kt index 3f312e0d2f1..c515e8d690d 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/label/PipelineViewDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/label/PipelineViewDao.kt @@ -29,6 +29,8 @@ package com.tencent.devops.process.dao.label import com.tencent.devops.model.process.tables.TPipelineView import com.tencent.devops.model.process.tables.records.TPipelineViewRecord +import com.tencent.devops.process.constant.PipelineViewType +import org.apache.commons.lang3.StringUtils import org.jooq.DSLContext import org.jooq.Result import org.springframework.stereotype.Repository @@ -89,7 +91,8 @@ class PipelineViewDao { isProject: Boolean, filters: String, userId: String, - id: Long? = null + id: Long? = null, + viewType: Int ): Long { with(TPipelineView.T_PIPELINE_VIEW) { val now = LocalDateTime.now() @@ -105,7 +108,8 @@ class PipelineViewDao { CREATE_TIME, UPDATE_TIME, CREATE_USER, - ID + ID, + VIEW_TYPE ) .values( projectId, @@ -118,7 +122,8 @@ class PipelineViewDao { now, now, userId, - id + id, + viewType ) .returning(ID) .fetchOne()!!.id @@ -153,16 +158,18 @@ class PipelineViewDao { name: String, logic: String, isProject: Boolean, - filters: String + filters: String, + viewType: Int ): Boolean { with(TPipelineView.T_PIPELINE_VIEW) { return dslContext.update(this) - .set(NAME, name) - .set(LOGIC, logic) + .let { if (StringUtils.isNotBlank(name)) it.set(NAME, name) else it } + .let { if (StringUtils.isNotBlank(logic)) it.set(LOGIC, logic) else it } + .let { if (filters.contains("@type")) it.set(FILTERS, filters) else it } + .let { if (viewType != PipelineViewType.UNCLASSIFIED) it.set(VIEW_TYPE, viewType) else it } .set(IS_PROJECT, isProject) .set(FILTER_BY_PIPEINE_NAME, "") .set(FILTER_BY_CREATOR, "") - .set(FILTERS, filters) .set(UPDATE_TIME, LocalDateTime.now()) .where(ID.eq(viewId).and(PROJECT_ID.eq(projectId))) .execute() == 1 @@ -189,6 +196,15 @@ class PipelineViewDao { } } + fun list(dslContext: DSLContext, projectId: String, viewType: Int): Result { + with(TPipelineView.T_PIPELINE_VIEW) { + return dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(VIEW_TYPE.eq(viewType)) + .fetch() + } + } + fun list(dslContext: DSLContext, projectId: String, isProject: Boolean): Result { with(TPipelineView.T_PIPELINE_VIEW) { return dslContext.selectFrom(this) @@ -227,6 +243,36 @@ class PipelineViewDao { } } + fun list( + dslContext: DSLContext, + userId: String, + projectId: String, + isProject: Boolean? = null, + viewType: Int? = null + ): List { + with(TPipelineView.T_PIPELINE_VIEW) { + return dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .let { + if (isProject == null) { + it.and(IS_PROJECT.eq(true).or(CREATE_USER.eq(userId))) + } else { + if (isProject) { + it.and(IS_PROJECT.eq(true)) + } else { + it.and(CREATE_USER.eq(userId)).and(IS_PROJECT.eq(false)) + } + } + }.let { + if (viewType == null) { + it + } else { + it.and(VIEW_TYPE.eq(viewType)) + } + }.fetch() + } + } + fun list( dslContext: DSLContext, projectId: String, @@ -241,7 +287,7 @@ class PipelineViewDao { } } - fun listProjectOrUser( + fun listAll( dslContext: DSLContext, projectId: String, isProject: Boolean, @@ -256,6 +302,45 @@ class PipelineViewDao { } } + fun listDynamicProjectId( + dslContext: DSLContext + ): List { + with(TPipelineView.T_PIPELINE_VIEW) { + return dslContext.select(PROJECT_ID).from(this) + .where(VIEW_TYPE.eq(PipelineViewType.DYNAMIC)) + .fetch(0, String::class.java) + .distinct() + } + } + + fun listDynamicViewByProjectId( + dslContext: DSLContext, + projectId: String + ): Result { + with(TPipelineView.T_PIPELINE_VIEW) { + return dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(VIEW_TYPE.eq(PipelineViewType.DYNAMIC)) + .fetch() + } + } + + fun listProjectOrUser( + dslContext: DSLContext, + projectId: String, + isProject: Boolean, + userId: String + ): Result { + with(TPipelineView.T_PIPELINE_VIEW) { + return dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(IS_PROJECT.eq(isProject)) + .let { if (isProject) it else it.and(CREATE_USER.eq(userId)) } + .orderBy(CREATE_TIME.desc()) + .fetch() + } + } + fun get(dslContext: DSLContext, projectId: String, viewId: Long): TPipelineViewRecord? { with(TPipelineView.T_PIPELINE_VIEW) { return dslContext.selectFrom(this) @@ -278,4 +363,55 @@ class PipelineViewDao { .fetchOne() } } + + fun countByName( + dslContext: DSLContext, + projectId: String, + name: String, + creator: String? = null, + isProject: Boolean, + excludeIds: Collection = emptySet() + ): Int { + with(TPipelineView.T_PIPELINE_VIEW) { + return dslContext.selectCount() + .from(this) + .where(PROJECT_ID.eq(projectId)) + .and(NAME.eq(name)) + .and(IS_PROJECT.eq(isProject)) + .let { if (null != creator) it.and(CREATE_USER.eq(creator)) else it } + .let { if (excludeIds.isNotEmpty()) it.and(ID.notIn(excludeIds)) else it } + .fetchOne()?.component1() ?: 0 + } + } + + fun fetchAnyByName( + dslContext: DSLContext, + projectId: String, + name: String, + isProject: Boolean + ): TPipelineViewRecord? { + with(TPipelineView.T_PIPELINE_VIEW) { + return dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(NAME.eq(name)) + .and(IS_PROJECT.eq(isProject)) + .fetchAny() + } + } + + fun countForLimit( + dslContext: DSLContext, + projectId: String, + isProject: Boolean, + userId: String + ): Int { + with(TPipelineView.T_PIPELINE_VIEW) { + return dslContext.selectCount() + .from(this) + .where(PROJECT_ID.eq(projectId)) + .and(IS_PROJECT.eq(isProject)) + .let { if (isProject) it else it.and(CREATE_USER.eq(userId)) } + .fetchOne()?.component1() ?: 0 + } + } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/label/PipelineViewGroupDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/label/PipelineViewGroupDao.kt new file mode 100644 index 00000000000..86e8be0afd2 --- /dev/null +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/label/PipelineViewGroupDao.kt @@ -0,0 +1,218 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.dao.label + +import com.tencent.devops.model.process.tables.TPipelineViewGroup +import com.tencent.devops.model.process.tables.records.TPipelineViewGroupRecord +import org.jooq.DSLContext +import org.jooq.impl.DSL.count +import org.springframework.stereotype.Repository +import java.time.LocalDateTime + +@Repository +class PipelineViewGroupDao { + fun create( + dslContext: DSLContext, + projectId: String, + viewId: Long, + pipelineId: String, + userId: String + ) { + with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + dslContext.insertInto( + this, + PROJECT_ID, + VIEW_ID, + PIPELINE_ID, + CREATE_TIME, + CREATOR + ).values( + projectId, + viewId, + pipelineId, + LocalDateTime.now(), + userId + ).onDuplicateKeyIgnore().execute() + } + } + + fun listByViewIds( + dslContext: DSLContext, + projectId: String, + viewIds: List + ): List { + return with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(VIEW_ID.`in`(viewIds)) + .fetch() + } + } + + fun listByViewId( + dslContext: DSLContext, + projectId: String, + viewId: Long + ): List { + return with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(VIEW_ID.eq(viewId)) + .fetch() + } + } + + fun listByProjectId( + dslContext: DSLContext, + projectId: String + ): List { + return with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .fetch() + } + } + + fun listByPipelineId( + dslContext: DSLContext, + projectId: String, + pipelineId: String + ): List { + return with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.eq(pipelineId)) + .fetch() + } + } + + fun listByPipelineIds( + dslContext: DSLContext, + projectId: String, + pipelineIds: Collection + ): List { + return with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.`in`(pipelineIds)) + .fetch() + } + } + + fun countByPipelineId( + dslContext: DSLContext, + projectId: String, + pipelineId: String + ) = with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + dslContext.selectCount().from(this) + .where(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.eq(pipelineId)) + .fetchOne()?.component1() ?: 0 + } + + fun remove( + dslContext: DSLContext, + projectId: String, + viewId: Long, + pipelineId: String + ) { + with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + dslContext.deleteFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(VIEW_ID.eq(viewId)) + .and(PIPELINE_ID.eq(pipelineId)) + .execute() + } + } + + fun remove( + dslContext: DSLContext, + projectId: String, + viewId: Long + ) { + with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + dslContext.deleteFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(VIEW_ID.eq(viewId)) + .execute() + } + } + + fun batchRemove( + dslContext: DSLContext, + projectId: String, + viewId: Long, + pipelineIds: List + ) { + with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + dslContext.deleteFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(VIEW_ID.eq(viewId)) + .and(PIPELINE_ID.`in`(pipelineIds)) + .execute() + } + } + + fun countByViewId( + dslContext: DSLContext, + projectId: String, + viewIds: Collection + ): Map { + with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + return dslContext.select(VIEW_ID, count()) + .from(this) + .where(PROJECT_ID.eq(projectId)) + .and(VIEW_ID.`in`(viewIds)) + .groupBy(VIEW_ID) + .fetch().map { it.value1() to it.value2() }.toMap() + } + } + + fun distinctPipelineIds( + dslContext: DSLContext, + projectId: String, + viewIds: Collection + ): List { + with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + return dslContext.selectDistinct(PIPELINE_ID) + .from(this) + .where(PROJECT_ID.eq(projectId)) + .and(VIEW_ID.`in`(viewIds)) + .fetch().getValues(0, String::class.java) + } + } + + fun delete(dslContext: DSLContext, projectId: String, pipelineId: String): Boolean { + with(TPipelineViewGroup.T_PIPELINE_VIEW_GROUP) { + return dslContext.deleteFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.eq(pipelineId)) + .execute() > 0 + } + } +} diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/label/PipelineViewTopDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/label/PipelineViewTopDao.kt new file mode 100644 index 00000000000..d151e939d8a --- /dev/null +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/label/PipelineViewTopDao.kt @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.dao.label + +import com.tencent.devops.model.process.tables.TPipelineViewTop +import com.tencent.devops.model.process.tables.records.TPipelineViewTopRecord +import org.jooq.DSLContext +import org.springframework.stereotype.Repository +import java.time.LocalDateTime + +@Repository +class PipelineViewTopDao { + fun add( + dslContext: DSLContext, + projectId: String, + viewId: Long, + userId: String + ) { + with(TPipelineViewTop.T_PIPELINE_VIEW_TOP) { + val now = LocalDateTime.now() + dslContext.insertInto( + this, + PROJECT_ID, + VIEW_ID, + CREATOR, + CREATE_TIME, + UPDATE_TIME + ).values( + projectId, + viewId, + userId, + now, + now + ).execute() + } + } + + fun remove( + dslContext: DSLContext, + projectId: String, + viewId: Long, + userId: String + ) { + with(TPipelineViewTop.T_PIPELINE_VIEW_TOP) { + dslContext.deleteFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(VIEW_ID.eq(viewId)) + .and(CREATOR.eq(userId)) + .execute() + } + } + + fun list( + dslContext: DSLContext, + projectId: String, + userId: String + ): List { + with(TPipelineViewTop.T_PIPELINE_VIEW_TOP) { + return dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(CREATOR.eq(userId)) + .orderBy(CREATE_TIME.desc()) + .fetch() + } + } +} diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildDao.kt index 30a9002f00d..102a445df5a 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildDao.kt @@ -33,11 +33,11 @@ import com.tencent.devops.common.api.pojo.ErrorInfo import com.tencent.devops.common.api.util.DateTimeUtil import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.timestampmilli -import com.tencent.devops.common.service.utils.JooqUtils import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.enums.StartType import com.tencent.devops.common.pipeline.pojo.BuildParameters +import com.tencent.devops.common.service.utils.JooqUtils import com.tencent.devops.model.process.Tables.T_PIPELINE_BUILD_HISTORY import com.tencent.devops.model.process.tables.TPipelineBuildHistory import com.tencent.devops.model.process.tables.records.TPipelineBuildHistoryRecord @@ -48,13 +48,13 @@ import com.tencent.devops.process.pojo.code.WebhookInfo import org.jooq.Condition import org.jooq.DSLContext import org.jooq.DatePart +import org.jooq.Record2 import org.jooq.Result +import org.jooq.SelectConditionStep import org.springframework.stereotype.Repository import java.sql.Timestamp import java.time.LocalDateTime import javax.ws.rs.core.Response -import org.jooq.Record2 -import org.jooq.SelectConditionStep @Suppress("ALL") @Repository @@ -181,7 +181,23 @@ class PipelineBuildDao { dslContext.select(PIPELINE_ID, BUILD_ID).from(this) .where(PROJECT_ID.eq(projectId)) .and(STATUS.`in`(statusSet.map { it.ordinal })) - .and(CONCURRENCY_GROUP.eq(concurrencyGroup)) + .and(CONCURRENCY_GROUP.eq(concurrencyGroup)).orderBy(START_TIME.asc()) + .fetch() + } + } + + fun getBuildTasksByConcurrencyGroupNull( + dslContext: DSLContext, + projectId: String, + pipelineId: String, + statusSet: List + ): List> { + return with(T_PIPELINE_BUILD_HISTORY) { + dslContext.select(PIPELINE_ID, BUILD_ID).from(this) + .where(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.eq(pipelineId)) + .and(STATUS.`in`(statusSet.map { it.ordinal })) + .and(CONCURRENCY_GROUP.isNull).orderBy(START_TIME.asc()) .fetch() } } @@ -838,6 +854,25 @@ class PipelineBuildDao { } } + fun getBuilds( + dslContext: DSLContext, + projectId: String, + pipelineId: String?, + buildStatus: Set? + ): List { + with(T_PIPELINE_BUILD_HISTORY) { + val dsl = dslContext.select(BUILD_ID).from(this) + .where(PROJECT_ID.eq(projectId)) + if (!pipelineId.isNullOrBlank()) { + dsl.and(PIPELINE_ID.eq(pipelineId)) + } + if (!buildStatus.isNullOrEmpty()) { + dsl.and(STATUS.`in`(buildStatus.map { it.ordinal })) + } + return dsl.fetch(BUILD_ID) + } + } + fun updateArtifactList( dslContext: DSLContext, artifactList: String?, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt index 53ffbdb4daa..210f5058489 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt @@ -37,16 +37,19 @@ import com.tencent.devops.model.process.tables.records.TPipelineBuildSummaryReco import com.tencent.devops.model.process.tables.records.TPipelineInfoRecord import com.tencent.devops.process.engine.pojo.LatestRunningBuild import com.tencent.devops.process.engine.pojo.PipelineFilterParam +import com.tencent.devops.process.pojo.PipelineCollation import com.tencent.devops.process.pojo.PipelineSortType import com.tencent.devops.process.pojo.classify.enums.Logic import com.tencent.devops.process.utils.PIPELINE_VIEW_ALL_PIPELINES import com.tencent.devops.process.utils.PIPELINE_VIEW_FAVORITE_PIPELINES +import com.tencent.devops.process.utils.PIPELINE_VIEW_MY_LIST_PIPELINES import com.tencent.devops.process.utils.PIPELINE_VIEW_MY_PIPELINES import org.jooq.Condition import org.jooq.DSLContext import org.jooq.Result import org.jooq.TableField import org.jooq.impl.DSL +import org.slf4j.LoggerFactory import org.springframework.stereotype.Repository import java.time.LocalDateTime @@ -153,7 +156,9 @@ class PipelineBuildSummaryDao { favorPipelines: List = emptyList(), authPipelines: List = emptyList(), pipelineFilterParamList: List? = null, - permissionFlag: Boolean? = null + permissionFlag: Boolean? = null, + includeDelete: Boolean? = false, + userId: String ): Long { val conditions = generatePipelineFilterCondition( projectId = projectId, @@ -163,7 +168,9 @@ class PipelineBuildSummaryDao { favorPipelines = favorPipelines, authPipelines = authPipelines, pipelineFilterParamList = pipelineFilterParamList, - permissionFlag = permissionFlag + permissionFlag = permissionFlag, + includeDelete = includeDelete, + userId = userId ) return dslContext.selectCount().from(T_PIPELINE_INFO) .where(conditions) @@ -183,7 +190,10 @@ class PipelineBuildSummaryDao { permissionFlag: Boolean? = null, page: Int? = null, pageSize: Int? = null, - pageOffsetNum: Int? = 0 + pageOffsetNum: Int? = 0, + includeDelete: Boolean? = false, + collation: PipelineCollation = PipelineCollation.DEFAULT, + userId: String? ): Result { val conditions = generatePipelineFilterCondition( projectId = projectId, @@ -193,7 +203,9 @@ class PipelineBuildSummaryDao { favorPipelines = favorPipelines, authPipelines = authPipelines, pipelineFilterParamList = pipelineFilterParamList, - permissionFlag = permissionFlag + permissionFlag = permissionFlag, + includeDelete = includeDelete, + userId = userId ) return listPipelineInfoBuildSummaryByConditions( dslContext = dslContext, @@ -202,7 +214,8 @@ class PipelineBuildSummaryDao { favorPipelines = favorPipelines, authPipelines = authPipelines, offset = page?.let { (it - 1) * (pageSize ?: 10) + (pageOffsetNum ?: 0) }, - limit = if (pageSize == -1) null else pageSize + limit = if (pageSize == -1) null else pageSize, + collation = collation ) } @@ -214,12 +227,16 @@ class PipelineBuildSummaryDao { favorPipelines: List, authPipelines: List, pipelineFilterParamList: List?, - permissionFlag: Boolean? + permissionFlag: Boolean?, + includeDelete: Boolean? = false, + userId: String? ): MutableList { val conditions = mutableListOf() conditions.add(T_PIPELINE_INFO.PROJECT_ID.eq(projectId)) conditions.add(T_PIPELINE_INFO.CHANNEL.eq(channelCode.name)) - conditions.add(T_PIPELINE_INFO.DELETE.eq(false)) + if (includeDelete == false) { + conditions.add(T_PIPELINE_INFO.DELETE.eq(false)) + } if (pipelineIds != null && pipelineIds.isNotEmpty()) { conditions.add(T_PIPELINE_INFO.PIPELINE_ID.`in`(pipelineIds)) } @@ -235,11 +252,14 @@ class PipelineBuildSummaryDao { } when (viewId) { PIPELINE_VIEW_FAVORITE_PIPELINES -> conditions.add(T_PIPELINE_INFO.PIPELINE_ID.`in`(favorPipelines)) - PIPELINE_VIEW_MY_PIPELINES -> conditions.add(T_PIPELINE_INFO.PIPELINE_ID.`in`(authPipelines)) + PIPELINE_VIEW_MY_PIPELINES -> if (userId != null) conditions.add(T_PIPELINE_INFO.CREATOR.eq(userId)) + PIPELINE_VIEW_MY_LIST_PIPELINES -> conditions.add(T_PIPELINE_INFO.PIPELINE_ID.`in`(authPipelines)) PIPELINE_VIEW_ALL_PIPELINES -> { // 查询所有流水线 } + else -> if (pipelineFilterParamList != null && pipelineFilterParamList.size > 1) { + logger.warn("this view logic has deprecated , viewId:$viewId") handleFilterParamCondition(pipelineFilterParamList[1], conditions) } } @@ -337,19 +357,24 @@ class PipelineBuildSummaryDao { com.tencent.devops.process.pojo.classify.enums.Condition.LIKE -> field.contains( fieldValue ) + com.tencent.devops.process.pojo.classify.enums.Condition.NOT_LIKE -> field.notLike( "%$fieldValue%" ) + com.tencent.devops.process.pojo.classify.enums.Condition.EQUAL -> field.eq( fieldValue ) + com.tencent.devops.process.pojo.classify.enums.Condition.NOT_EQUAL -> field.ne( fieldValue ) + com.tencent.devops.process.pojo.classify.enums.Condition.INCLUDE -> JooqUtils.strPosition( field, fieldValue ).gt(0) + com.tencent.devops.process.pojo.classify.enums.Condition.NOT_INCLUDE -> JooqUtils.strPosition( field, fieldValue @@ -367,22 +392,50 @@ class PipelineBuildSummaryDao { favorPipelines: List = emptyList(), authPipelines: List = emptyList(), offset: Int? = null, - limit: Int? = null + limit: Int? = null, + collation: PipelineCollation ): Result { val baseStep = dslContext.selectFrom(T_PIPELINE_INFO).where(conditions) if (sortType != null) { val sortTypeField = when (sortType) { PipelineSortType.NAME -> { - T_PIPELINE_INFO.PIPELINE_NAME_PINYIN.asc() + T_PIPELINE_INFO.PIPELINE_NAME_PINYIN.let { + if (collation == PipelineCollation.DEFAULT || collation == PipelineCollation.ASC) { + it.asc() + } else { + it.desc() + } + } } + PipelineSortType.CREATE_TIME -> { - T_PIPELINE_INFO.CREATE_TIME.desc() + T_PIPELINE_INFO.CREATE_TIME.let { + if (collation == PipelineCollation.DEFAULT || collation == PipelineCollation.DESC) { + it.desc() + } else { + it.asc() + } + } } + PipelineSortType.UPDATE_TIME -> { - T_PIPELINE_INFO.UPDATE_TIME.desc() + T_PIPELINE_INFO.UPDATE_TIME.let { + if (collation == PipelineCollation.DEFAULT || collation == PipelineCollation.DESC) { + it.desc() + } else { + it.asc() + } + } } + PipelineSortType.LAST_EXEC_TIME -> { - T_PIPELINE_INFO.LATEST_START_TIME.desc() + T_PIPELINE_INFO.LATEST_START_TIME.let { + if (collation == PipelineCollation.DEFAULT || collation == PipelineCollation.DESC) { + it.desc() + } else { + it.asc() + } + } } } baseStep.orderBy(sortTypeField, T_PIPELINE_INFO.PIPELINE_ID) @@ -563,4 +616,8 @@ class PipelineBuildSummaryDao { .execute() == 1 } } + + companion object { + private val logger = LoggerFactory.getLogger(PipelineBuildSummaryDao::class.java) + } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt index 858a2959f30..13a1fc29afa 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt @@ -40,8 +40,10 @@ import com.tencent.devops.process.engine.pojo.UpdateTaskInfo import com.tencent.devops.process.utils.PIPELINE_TASK_MESSAGE_STRING_LENGTH_MAX import org.jooq.DSLContext import org.jooq.Record1 +import org.jooq.Record3 import org.jooq.RecordMapper import org.jooq.Result +import org.jooq.impl.DSL.count import org.slf4j.LoggerFactory import org.springframework.stereotype.Repository import java.time.Duration @@ -352,6 +354,21 @@ class PipelineBuildTaskDao { } } + fun countGroupByBuildId( + dslContext: DSLContext, + projectId: String, + buildIds: Collection + ): Result> { + with(TPipelineBuildTask.T_PIPELINE_BUILD_TASK) { + return dslContext.select(BUILD_ID, STATUS, count()) + .from(this) + .where(PROJECT_ID.eq(projectId)) + .and(BUILD_ID.`in`(buildIds)) + .groupBy(BUILD_ID, STATUS) + .fetch() + } + } + fun updateTaskParam( dslContext: DSLContext, projectId: String, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt index 385fe53ab6c..0cd2d10cb13 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt @@ -33,6 +33,8 @@ import com.tencent.devops.common.util.PinyinUtil import com.tencent.devops.model.process.Tables.T_PIPELINE_INFO import com.tencent.devops.model.process.tables.records.TPipelineInfoRecord import com.tencent.devops.process.engine.pojo.PipelineInfo +import com.tencent.devops.process.pojo.PipelineCollation +import com.tencent.devops.process.pojo.PipelineSortType import org.jooq.Condition import org.jooq.DSLContext import org.jooq.Record1 @@ -166,7 +168,7 @@ class PipelineInfoDao { } logger.info( "Update the pipeline $pipelineId add new version($version) old version($latestVersion) " + - "and result=${count == 1}" + "and result=${count == 1}" ) return version } @@ -244,15 +246,21 @@ class PipelineInfoDao { projectId: String? = null, limit: Int, offset: Int, - deleteFlag: Boolean = false, - timeDescFlag: Boolean = true + deleteFlag: Boolean? = false, + timeDescFlag: Boolean = true, + channelCode: ChannelCode? = null ): Result? { return with(T_PIPELINE_INFO) { val conditions = mutableListOf() if (projectId != null) { conditions.add(PROJECT_ID.eq(projectId)) } - conditions.add(DELETE.eq(deleteFlag)) + if (null != deleteFlag) { + conditions.add(DELETE.eq(deleteFlag)) + } + if (null != channelCode) { + conditions.add(CHANNEL.eq(channelCode.name)) + } val baseQuery = dslContext.selectFrom(this).where(conditions) if (timeDescFlag) { baseQuery.orderBy(CREATE_TIME.desc(), PIPELINE_ID) @@ -310,10 +318,15 @@ class PipelineInfoDao { } } + @SuppressWarnings("ComplexMethod") fun listDeletePipelineIdByProject( dslContext: DSLContext, projectId: String, - days: Long? + days: Long?, + offset: Int? = null, + limit: Int? = null, + sortType: PipelineSortType, + collation: PipelineCollation ): Result? { with(T_PIPELINE_INFO) { val conditions = mutableListOf() @@ -322,8 +335,43 @@ class PipelineInfoDao { if (days != null) { conditions.add(UPDATE_TIME.greaterOrEqual(LocalDateTime.now().minusDays(days))) } - return dslContext.selectFrom(this) - .where(conditions).fetch() + return dslContext + .selectFrom(this) + .where(conditions) + .let { + val st = when (sortType) { + PipelineSortType.UPDATE_TIME -> UPDATE_TIME + PipelineSortType.NAME -> PIPELINE_NAME_PINYIN + else -> CREATE_TIME + } + val c = if (collation == PipelineCollation.DEFAULT || collation == PipelineCollation.DESC) { + st.desc() + } else { + st.asc() + } + it.orderBy(c) + } + .let { if (offset != null && limit != null) it.limit(offset, limit) else it } + .fetch() + } + } + + fun countDeletePipeline( + dslContext: DSLContext, + projectId: String, + days: Long? + ): Int { + with(T_PIPELINE_INFO) { + val conditions = mutableListOf() + conditions.add(PROJECT_ID.eq(projectId)) + conditions.add(DELETE.eq(true)) + if (days != null) { + conditions.add(UPDATE_TIME.greaterOrEqual(LocalDateTime.now().minusDays(days))) + } + return dslContext + .selectCount().from(this) + .where(conditions) + .fetchOne()?.value1() ?: 0 } } @@ -606,10 +654,10 @@ class PipelineInfoDao { } } - fun listByProject(dslContext: DSLContext, projectCode: String): Result> { + fun listByProject(dslContext: DSLContext, projectId: String): Result> { return with(T_PIPELINE_INFO) { dslContext.select(PIPELINE_ID.`as`("pipelineId"), ID.`as`("id")).from(this) - .where(PROJECT_ID.eq(projectCode).and(DELETE.eq(false))).fetch() + .where(PROJECT_ID.eq(projectId).and(DELETE.eq(false))).fetch() } } @@ -625,6 +673,24 @@ class PipelineInfoDao { } } + fun countExcludePipelineIds( + dslContext: DSLContext, + projectId: String, + excludePipelineIds: List, + channelCode: ChannelCode? = null, + includeDelete: Boolean = false + ): Int { + with(T_PIPELINE_INFO) { + return dslContext.selectCount() + .from(this) + .where(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.notIn(excludePipelineIds)) + .let { if (channelCode == null) it else it.and(CHANNEL.eq(channelCode.name)) } + .let { if (includeDelete) it else it.and(DELETE.eq(false)) } + .fetchOne()?.value1() ?: 0 + } + } + fun getPipelineByAutoId( dslContext: DSLContext, ids: List, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/QueueInterceptor.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/QueueInterceptor.kt index 1aeb8f2008d..a52ab9fda52 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/QueueInterceptor.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/QueueInterceptor.kt @@ -253,9 +253,9 @@ class QueueInterceptor @Autowired constructor( status = PIPELINE_SETTING_NOT_EXISTS.toInt(), message = "流水线设置不存在/Setting not found" ) - val concurrencyGroup = setting.concurrencyGroup + val concurrencyGroup = setting.concurrencyGroup ?: task.pipelineInfo.pipelineId return when { - !concurrencyGroup.isNullOrBlank() -> { + concurrencyGroup.isNotBlank() -> { if (setting.concurrencyCancelInProgress) { val detailUrl = pipelineUrlBean.genBuildDetailUrl( projectCode = projectId, @@ -266,12 +266,25 @@ class QueueInterceptor @Autowired constructor( needShortUrl = false ) // cancel-in-progress: true时， 若有相同 group 的流水线正在执行，则取消正在执行的流水线，新来的触发开始执行 - val status = listOf(BuildStatus.RUNNING, BuildStatus.QUEUE) - pipelineRuntimeService.getBuildInfoListByConcurrencyGroup( + // status 取所有没有完成的状态 + val status = BuildStatus.values().filterNot { it.isFinish() } + val builds = pipelineRuntimeService.getBuildInfoListByConcurrencyGroup( projectId = projectId, concurrencyGroup = concurrencyGroup, status = status - ).forEach { (pipelineId, buildId) -> + ).toMutableList() + // #8143 兼容旧流水线版本 TODO 待模板设置补上漏洞，后期下掉 # 8143 + if (concurrencyGroup == task.pipelineInfo.pipelineId) { + builds.addAll( + 0, + pipelineRuntimeService.getBuildInfoListByConcurrencyGroupNull( + projectId = projectId, + pipelineId = task.pipelineInfo.pipelineId, + status = status + ) + ) + } + builds.forEach { (pipelineId, buildId) -> cancelBuildPipeline( projectId = projectId, pipelineId = pipelineId, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/RunLockInterceptor.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/RunLockInterceptor.kt index 9fe3620d3c3..5ebf5b174f4 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/RunLockInterceptor.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/RunLockInterceptor.kt @@ -72,12 +72,23 @@ class RunLockInterceptor @Autowired constructor( Response(BuildStatus.RUNNING) } } else if (runLockType == PipelineRunLockType.GROUP_LOCK) { - val concurrencyGroupRunningCount = concurrencyGroup?.let { - pipelineRuntimeService.getBuildInfoListByConcurrencyGroup( + val concurrencyGroupNotNull = concurrencyGroup ?: pipelineId + val concurrencyGroupRunningCount = concurrencyGroupNotNull.let { + var size = pipelineRuntimeService.getBuildInfoListByConcurrencyGroup( projectId = projectId, concurrencyGroup = it, status = listOf(BuildStatus.RUNNING) ).size + + // #8143 兼容旧流水线版本 TODO 待模板设置补上漏洞，后期下掉 # 8143 + if (it == pipelineId) { + size += pipelineRuntimeService.getBuildInfoListByConcurrencyGroupNull( + projectId = projectId, + pipelineId = pipelineId, + status = listOf(BuildStatus.RUNNING) + ).size + } + return@let size } ?: 0 if (concurrencyGroupRunningCount >= 1) { logger.info("[$pipelineId] 当前互斥组[$concurrencyGroup]同时只能运行一个构建任务，开始排队！") diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/TimerTriggerScmChangeInterceptor.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/TimerTriggerScmChangeInterceptor.kt index 39f600513b0..201b1925ac6 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/TimerTriggerScmChangeInterceptor.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/TimerTriggerScmChangeInterceptor.kt @@ -29,6 +29,7 @@ package com.tencent.devops.process.engine.interceptor import com.tencent.devops.common.api.enums.RepositoryConfig import com.tencent.devops.common.api.enums.RepositoryType +import com.tencent.devops.common.api.enums.RepositoryTypeNew import com.tencent.devops.common.api.util.EnvUtils import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.container.TriggerContainer @@ -418,6 +419,8 @@ class TimerTriggerScmChangeInterceptor @Autowired constructor( val input = ele.data["input"] if (input !is Map<*, *>) return false + // checkout插件[按仓库URL输入]不校验代码变更 + if (ele.getAtomCode() == "checkout" && input["repositoryType"] == RepositoryTypeNew.URL.name) return true val repositoryConfig = getMarketBuildRepoConfig(input, variables) ?: return false val gitPullMode = EnvUtils.parseEnv(input["pullType"] as String?, variables) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt index 6ad4acb6f8e..db5e05c3fc9 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt @@ -30,6 +30,7 @@ package com.tencent.devops.process.engine.service import com.tencent.devops.common.api.exception.DependNotFoundException import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.exception.InvalidParamException +import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings import com.tencent.devops.common.api.util.DateTimeUtil import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.client.Client @@ -53,6 +54,7 @@ import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.process.constant.ProcessMessageCode import com.tencent.devops.process.dao.PipelineSettingDao import com.tencent.devops.process.dao.PipelineSettingVersionDao +import com.tencent.devops.process.dao.label.PipelineViewGroupDao import com.tencent.devops.process.engine.cfg.ModelContainerIdGenerator import com.tencent.devops.process.engine.cfg.ModelTaskIdGenerator import com.tencent.devops.process.engine.cfg.PipelineIdGenerator @@ -71,10 +73,11 @@ import com.tencent.devops.process.engine.pojo.event.PipelineDeleteEvent import com.tencent.devops.process.engine.pojo.event.PipelineRestoreEvent import com.tencent.devops.process.engine.pojo.event.PipelineUpdateEvent import com.tencent.devops.process.plugin.load.ElementBizRegistrar +import com.tencent.devops.process.pojo.PipelineCollation +import com.tencent.devops.process.pojo.PipelineSortType import com.tencent.devops.process.pojo.pipeline.DeletePipelineResult import com.tencent.devops.process.pojo.pipeline.DeployPipelineResult import com.tencent.devops.process.pojo.pipeline.PipelineSubscriptionType -import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings import com.tencent.devops.process.pojo.setting.PipelineModelVersion import com.tencent.devops.process.pojo.setting.PipelineRunLockType import com.tencent.devops.process.pojo.setting.PipelineSetting @@ -116,6 +119,7 @@ class PipelineRepositoryService constructor( private val templatePipelineDao: TemplatePipelineDao, private val pipelineResVersionDao: PipelineResVersionDao, private val pipelineSettingVersionDao: PipelineSettingVersionDao, + private val pipelineViewGroupDao: PipelineViewGroupDao, private val versionConfigure: VersionConfigure, private val pipelineInfoExtService: PipelineInfoExtService, private val client: Client, @@ -268,12 +272,12 @@ class PipelineRepositoryService constructor( ) } val c = ( - stage.containers.getOrNull(0) - ?: throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_PIPELINE_MODEL_NEED_JOB, - defaultMessage = "第一阶段的环境不能为空" - ) - ) as TriggerContainer + stage.containers.getOrNull(0) + ?: throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_MODEL_NEED_JOB, + defaultMessage = "第一阶段的环境不能为空" + ) + ) as TriggerContainer // #4518 各个容器ID的初始化 c.id = containerSeqId.get().toString() @@ -373,9 +377,11 @@ class PipelineRepositoryService constructor( c.matrixGroupFlag != true -> { // c.matrixGroupFlag 不为 true 时 不需要做yaml检查 } + c is NormalContainer -> { matrixYamlCheck(c.matrixControlOption) } + c is VMBuildContainer -> { matrixYamlCheck(c.matrixControlOption) } @@ -446,8 +452,8 @@ class PipelineRepositoryService constructor( if ((option.maxConcurrency ?: 0) > PIPELINE_MATRIX_CON_RUNNING_SIZE_MAX) { throw InvalidParamException( "构建矩阵并发数(${option.maxConcurrency}) 超过 $PIPELINE_MATRIX_CON_RUNNING_SIZE_MAX /" + - "matrix maxConcurrency(${option.maxConcurrency}) " + - "is larger than $PIPELINE_MATRIX_CON_RUNNING_SIZE_MAX" + "matrix maxConcurrency(${option.maxConcurrency}) " + + "is larger than $PIPELINE_MATRIX_CON_RUNNING_SIZE_MAX" ) } MatrixContextUtils.schemaCheck( @@ -836,6 +842,7 @@ class PipelineRepositoryService constructor( pipelineResDao.deleteAllVersion(transactionContext, projectId, pipelineId) pipelineSettingDao.delete(transactionContext, projectId, pipelineId) templatePipelineDao.delete(transactionContext, projectId, pipelineId) + pipelineViewGroupDao.delete(transactionContext, projectId, pipelineId) } else { // 删除前改名，防止名称占用 val deleteTime = LocalDateTime.now().toString("yyMMddHHmmSS") @@ -1125,8 +1132,23 @@ class PipelineRepositoryService constructor( /** * 列出已经删除的流水线 */ - fun listDeletePipelineIdByProject(projectId: String, days: Long?): List { - val result = pipelineInfoDao.listDeletePipelineIdByProject(dslContext, projectId, days) + fun listDeletePipelineIdByProject( + projectId: String, + days: Long?, + offset: Int? = null, + limit: Int? = null, + sortType: PipelineSortType, + collation: PipelineCollation + ): List { + val result = pipelineInfoDao.listDeletePipelineIdByProject( + dslContext = dslContext, + projectId = projectId, + days = days, + offset = offset, + limit = limit, + sortType = sortType, + collation = collation + ) val list = mutableListOf() result?.forEach { if (it != null) { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt index 24c095bbbd6..40b3eebd11f 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt @@ -73,11 +73,16 @@ import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.service.trace.TraceTag import com.tencent.devops.common.service.utils.MessageCodeUtil import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_EVENT_TYPE +import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_ISSUE_IID import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_MR_ID import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_MR_MERGE_COMMIT_SHA import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_MR_NUMBER import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_MR_SOURCE_BRANCH import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_MR_URL +import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_NOTE_ID +import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_REVIEW_ID +import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_TAG_NAME +import com.tencent.devops.common.webhook.pojo.code.BK_REPO_WEBHOOK_REPO_ALIAS_NAME import com.tencent.devops.common.webhook.pojo.code.BK_REPO_WEBHOOK_REPO_AUTH_USER import com.tencent.devops.common.webhook.pojo.code.BK_REPO_WEBHOOK_REPO_URL import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_BRANCH @@ -245,6 +250,9 @@ class PipelineRuntimeService @Autowired constructor( return pipelineBuildDao.convert(t) } + /** 根据状态信息获取并发组构建列表 + * @return Pair( PIPELINE_ID , BUILD_ID ) + */ fun getBuildInfoListByConcurrencyGroup( projectId: String, concurrencyGroup: String, @@ -258,6 +266,19 @@ class PipelineRuntimeService @Autowired constructor( ).map { Pair(it.value1(), it.value2()) } } + fun getBuildInfoListByConcurrencyGroupNull( + projectId: String, + pipelineId: String, + status: List + ): List> { + return pipelineBuildDao.getBuildTasksByConcurrencyGroupNull( + dslContext = dslContext, + projectId = projectId, + pipelineId = pipelineId, + statusSet = status + ).map { Pair(it.value1(), it.value2()) } + } + fun getBuildNoByByPair(buildIds: Set, projectId: String?): MutableMap { val result = mutableMapOf() val buildInfoList = pipelineBuildDao.listBuildInfoByBuildIds( @@ -300,7 +321,8 @@ class PipelineRuntimeService @Autowired constructor( pipelineFilterParamList = pipelineFilterParamList, permissionFlag = permissionFlag, page = page, - pageSize = pageSize + pageSize = pageSize, + userId = null ) } @@ -1297,6 +1319,7 @@ class PipelineRuntimeService @Autowired constructor( webhookRepoUrl = params[BK_REPO_WEBHOOK_REPO_URL]?.toString(), webhookType = params[PIPELINE_WEBHOOK_TYPE]?.toString(), webhookBranch = params[PIPELINE_WEBHOOK_BRANCH]?.toString(), + webhookAliasName = params[BK_REPO_WEBHOOK_REPO_ALIAS_NAME]?.toString(), // GIT事件分为MR和MR accept,但是PIPELINE_WEBHOOK_EVENT_TYPE值只有MR webhookEventType = if (params[PIPELINE_WEBHOOK_TYPE] == CodeType.GIT.name) { params[BK_REPO_GIT_WEBHOOK_EVENT_TYPE]?.toString() @@ -1309,7 +1332,11 @@ class PipelineRuntimeService @Autowired constructor( mrId = params[BK_REPO_GIT_WEBHOOK_MR_ID]?.toString(), mrIid = params[BK_REPO_GIT_WEBHOOK_MR_NUMBER]?.toString(), mrUrl = params[BK_REPO_GIT_WEBHOOK_MR_URL]?.toString(), - repoAuthUser = params[BK_REPO_WEBHOOK_REPO_AUTH_USER]?.toString() + repoAuthUser = params[BK_REPO_WEBHOOK_REPO_AUTH_USER]?.toString(), + tagName = params[BK_REPO_GIT_WEBHOOK_TAG_NAME]?.toString(), + issueIid = params[BK_REPO_GIT_WEBHOOK_ISSUE_IID]?.toString(), + noteId = params[BK_REPO_GIT_WEBHOOK_NOTE_ID]?.toString(), + reviewId = params[BK_REPO_GIT_WEBHOOK_REVIEW_ID]?.toString() ), formatted = false ) @@ -1635,29 +1662,42 @@ class PipelineRuntimeService @Autowired constructor( return pipelineBuildDao.count(dslContext = dslContext, projectId = projectId, pipelineId = pipelineId) } + fun getBuilds( + projectId: String, + pipelineId: String?, + buildStatus: Set? + ): List { + return pipelineBuildDao.getBuilds( + dslContext = dslContext, + projectId = projectId, + pipelineId = pipelineId, + buildStatus = buildStatus + ) + } + fun getPipelineBuildHistoryCount( projectId: String, pipelineId: String, - materialAlias: List?, - materialUrl: String?, - materialBranch: List?, - materialCommitId: String?, - materialCommitMessage: String?, + materialAlias: List? = null, + materialUrl: String? = null, + materialBranch: List? = null, + materialCommitId: String? = null, + materialCommitMessage: String? = null, status: List?, - trigger: List?, - queueTimeStartTime: Long?, - queueTimeEndTime: Long?, - startTimeStartTime: Long?, - startTimeEndTime: Long?, - endTimeStartTime: Long?, - endTimeEndTime: Long?, - totalTimeMin: Long?, - totalTimeMax: Long?, - remark: String?, - buildNoStart: Int?, - buildNoEnd: Int?, - buildMsg: String?, - startUser: List? + trigger: List? = null, + queueTimeStartTime: Long? = null, + queueTimeEndTime: Long? = null, + startTimeStartTime: Long? = null, + startTimeEndTime: Long? = null, + endTimeStartTime: Long? = null, + endTimeEndTime: Long? = null, + totalTimeMin: Long? = null, + totalTimeMax: Long? = null, + remark: String? = null, + buildNoStart: Int? = null, + buildNoEnd: Int? = null, + buildMsg: String? = null, + startUser: List? = null ): Int { return pipelineBuildDao.count( dslContext = dslContext, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt index e9491513e6f..bad7a74a382 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt @@ -197,6 +197,11 @@ class PipelineStageService @Autowired constructor( fun pauseStage(buildStage: PipelineBuildStage) { with(buildStage) { + // 兜底保护，若已经被审核过则直接忽略 + if (checkIn?.status == BuildStatus.REVIEW_ABORT.name || + checkIn?.status == BuildStatus.REVIEW_PROCESSED.name) { + return@with + } checkIn?.status = BuildStatus.REVIEWING.name val allStageStatus = stageBuildDetailService.stagePause( projectId = projectId, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/BaseBuildDetailService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/BaseBuildDetailService.kt index 12babe6750a..b7bb067d692 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/BaseBuildDetailService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/BaseBuildDetailService.kt @@ -33,6 +33,7 @@ import com.tencent.devops.common.api.constant.BUILD_COMPLETED import com.tencent.devops.common.api.constant.BUILD_FAILED import com.tencent.devops.common.api.constant.BUILD_REVIEWING import com.tencent.devops.common.api.constant.BUILD_RUNNING +import com.tencent.devops.common.api.constant.BUILD_STAGE_SUCCESS import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.Watcher import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher @@ -157,16 +158,14 @@ open class BaseBuildDetailService constructor( val stageTagMap: Map by lazy { stageTagService.getAllStageTag().data!!.associate { it.id to it.stageTagName } } // 更新Stage状态至BuildHistory - val (statusMessage, reason) = if (buildStatus == BuildStatus.REVIEWING) { - Pair(BUILD_REVIEWING, reviewers?.joinToString(",")) - } else if (buildStatus.isFailure()) { - Pair(BUILD_FAILED, errorMsg ?: buildStatus.name) - } else if (buildStatus.isCancel()) { - Pair(BUILD_CANCELED, cancelUser) - } else if (buildStatus.isSuccess()) { - Pair(BUILD_COMPLETED, null) - } else { - Pair(BUILD_RUNNING, null) + + val (statusMessage, reason) = when { + buildStatus == BuildStatus.REVIEWING -> Pair(BUILD_REVIEWING, reviewers?.joinToString(",")) + buildStatus == BuildStatus.STAGE_SUCCESS -> Pair(BUILD_STAGE_SUCCESS, null) + buildStatus.isFailure() -> Pair(BUILD_FAILED, errorMsg ?: buildStatus.name) + buildStatus.isCancel() -> Pair(BUILD_CANCELED, cancelUser) + buildStatus.isSuccess() -> Pair(BUILD_COMPLETED, null) + else -> Pair(BUILD_RUNNING, null) } return model.stages.map { BuildStageStatus( diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/TaskBuildDetailService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/TaskBuildDetailService.kt index eba9a032386..336f8191cf4 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/TaskBuildDetailService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/TaskBuildDetailService.kt @@ -353,27 +353,43 @@ class TaskBuildDetailService( elements: List, updateTaskStatusInfos: MutableList? ): Boolean { - if (cancelTaskPostFlag) { - return handleCancelTaskPost( - containerId = containerId, - endElement = endElement, - endElementIndex = endElementIndex, - tmpElement = tmpElement, - tmpElementIndex = tmpElementIndex, - elements = elements, - updateTaskStatusInfos = updateTaskStatusInfos - ) - } else { - return handleCancelTaskNormal( - tmpElement = tmpElement, - endElement = endElement, - buildStatus = buildStatus, - endElementIndex = endElementIndex, - elements = elements, - containerId = containerId, - updateTaskStatusInfos = updateTaskStatusInfos - ) + when { + cancelTaskPostFlag -> { + return handleCancelTaskPost( + containerId = containerId, + endElement = endElement, + endElementIndex = endElementIndex, + tmpElement = tmpElement, + tmpElementIndex = tmpElementIndex, + elements = elements, + updateTaskStatusInfos = updateTaskStatusInfos + ) + } + buildStatus.isCancel() -> { + return handleCancelTaskNormal( + tmpElement = tmpElement, + endElement = endElement, + buildStatus = buildStatus, + endElementIndex = endElementIndex, + elements = elements, + containerId = containerId, + updateTaskStatusInfos = updateTaskStatusInfos + ) + } + buildStatus.isSkip() -> { + updateTaskStatusInfos?.add( + PipelineTaskStatusInfo( + taskId = endElement.id!!, + containerHashId = containerId, + buildStatus = buildStatus, + executeCount = endElement.executeCount, + message = endElement.errorMsg + ) + ) + return false + } } + return false } private fun handleCancelTaskNormal( diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/label/PipelineGroupService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/label/PipelineGroupService.kt index c2a59b6b04e..6ac7393419d 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/label/PipelineGroupService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/label/PipelineGroupService.kt @@ -29,12 +29,12 @@ package com.tencent.devops.process.service.label import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.exception.OperationException -import com.tencent.devops.common.event.enums.PipelineLabelChangeTypeEnum -import com.tencent.devops.common.event.pojo.measure.PipelineLabelRelateInfo import com.tencent.devops.common.api.util.HashUtil import com.tencent.devops.common.api.util.timestamp import com.tencent.devops.common.client.Client +import com.tencent.devops.common.event.enums.PipelineLabelChangeTypeEnum import com.tencent.devops.common.event.pojo.measure.LabelChangeMetricsBroadCastEvent +import com.tencent.devops.common.event.pojo.measure.PipelineLabelRelateInfo import com.tencent.devops.model.process.tables.records.TPipelineFavorRecord import com.tencent.devops.model.process.tables.records.TPipelineGroupRecord import com.tencent.devops.model.process.tables.records.TPipelineLabelRecord @@ -330,18 +330,14 @@ class PipelineGroupService @Autowired constructor( } try { val labelIdArr = labelIds.map { decode(it) }.toSet() - val pipelineLabelRels = mutableListOf>() - labelIdArr.forEach { labelId -> - val id = client.get(ServiceAllocIdResource::class) - .generateSegmentId(PIPELINE_LABEL_PIPELINE_BIZ_TAG_NAME).data - pipelineLabelRels.add(Pair(labelId, id)) - } + val pipelineLabelSegmentIdPairs = pipelineLabelSegmentIdPairs(labelIdArr) pipelineLabelPipelineDao.batchCreate( dslContext = dslContext, projectId = projectId, pipelineId = pipelineId, - pipelineLabelRels = pipelineLabelRels, - userId = userId) + pipelineLabelRels = pipelineLabelSegmentIdPairs, + userId = userId + ) val createData = pipelineLabelDao.getByIds(dslContext, projectId, labelIdArr) measureEventDispatcher.dispatch( @@ -371,12 +367,7 @@ class PipelineGroupService @Autowired constructor( fun updatePipelineLabel(userId: String, projectId: String, pipelineId: String, labelIds: List) { val labelIdArr = labelIds.map { decode(it) }.toSet() - val pipelineLabelRels = mutableListOf>() - labelIdArr.forEach { labelId -> - val id = - client.get(ServiceAllocIdResource::class).generateSegmentId(PIPELINE_LABEL_PIPELINE_BIZ_TAG_NAME).data - pipelineLabelRels.add(Pair(labelId, id)) - } + val pipelineLabelSegmentIdPairs = pipelineLabelSegmentIdPairs(labelIdArr) try { dslContext.transaction { configuration -> val context = DSL.using(configuration) @@ -390,7 +381,7 @@ class PipelineGroupService @Autowired constructor( dslContext = context, projectId = projectId, pipelineId = pipelineId, - pipelineLabelRels = pipelineLabelRels, + pipelineLabelRels = pipelineLabelSegmentIdPairs, userId = userId ) } @@ -430,7 +421,17 @@ class PipelineGroupService @Autowired constructor( } ) ) - logger.info("LableChangeMetricsBroadCastEvent： updatePipelineLabel-create $projectId|$pipelineId|$labelIdArr|$labelIds") + logger.info("LableChangeMetricsBroadCastEvent： " + + "updatePipelineLabel-create $projectId|$pipelineId|$labelIdArr|$labelIds") + } + + private fun pipelineLabelSegmentIdPairs(labelIdArr: Set): MutableList> { + val generateSegmentIds = client.get(ServiceAllocIdResource::class) + .batchGenerateSegmentId(PIPELINE_LABEL_PIPELINE_BIZ_TAG_NAME, labelIdArr.size) + val pairs = mutableListOf>() + var index = 0 + labelIdArr.forEach { pairs.add(Pair(it, generateSegmentIds.data!![index++])) } + return pairs } fun getViewLabelToPipelinesMap(projectId: String, labels: List): Map> { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt index 384ad880b81..457f33a1c32 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt @@ -239,6 +239,8 @@ class PipelineBuildService( pipelineParamMap[PIPELINE_BUILD_MSG]?.let { buildMsgParam -> originStartParams.add(buildMsgParam) } pipelineParamMap[PIPELINE_RETRY_COUNT]?.let { retryCountParam -> originStartParams.add(retryCountParam) } + val buildId = pipelineParamMap[PIPELINE_RETRY_BUILD_ID]?.value?.toString() ?: buildIdGenerator.getNextId() + // #6987 修复stream的并发执行判断问题 在判断并发时再替换上下文 setting?.concurrencyGroup?.let { val varMap = pipelineParamMap.values.associate { param -> param.key to param.value.toString() } @@ -246,8 +248,6 @@ class PipelineBuildService( logger.info("[$pipelineId]|Concurrency Group is ${setting.concurrencyGroup}") } - val buildId = pipelineParamMap[PIPELINE_RETRY_BUILD_ID]?.value?.toString() ?: buildIdGenerator.getNextId() - val interceptResult = pipelineInterceptorChain.filter( InterceptData( pipelineInfo = pipeline, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineStatusService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineStatusService.kt index d763f1478c0..9a1c0e2468f 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineStatusService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineStatusService.kt @@ -32,6 +32,7 @@ import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.utils.BuildStatusSwitcher import com.tencent.devops.process.dao.PipelineSettingDao +import com.tencent.devops.process.engine.dao.PipelineBuildTaskDao import com.tencent.devops.process.engine.dao.PipelineInfoDao import com.tencent.devops.process.engine.service.PipelineRuntimeService import com.tencent.devops.process.pojo.PipelineStatus @@ -45,6 +46,7 @@ class PipelineStatusService( private val dslContext: DSLContext, private val pipelineInfoDao: PipelineInfoDao, private val pipelineSettingDao: PipelineSettingDao, + private val pipelineBuildTaskDao: PipelineBuildTaskDao, private val pipelineRuntimeService: PipelineRuntimeService ) { @@ -64,6 +66,16 @@ class PipelineStatusService( val pipelineBuildStatus = getBuildStatus(buildStatusOrd) + // 获取构建执行进度 + val buildTaskCountList = pipelineBuildTaskDao.countGroupByBuildId( + dslContext = dslContext, + projectId = projectId, + buildIds = listOf(pipelineBuildSummary.latestBuildId) + ) + val lastBuildTotalCount = buildTaskCountList.sumOf { it.value3() } + val lastBuildFinishCount = + buildTaskCountList.filter { it.value2() == BuildStatus.SUCCEED.ordinal }.sumOf { it.value3() } + // todo还没想好与Pipeline结合，减少这部分的代码，收归一处 return PipelineStatus( taskCount = pipelineInfo.taskCount, @@ -79,7 +91,9 @@ class PipelineStatusService( latestBuildStatus = pipelineBuildStatus, latestBuildTaskName = pipelineBuildSummary.latestTaskName, lock = PipelineRunLockType.checkLock(pipelineSetting.runLockType), - runningBuildCount = pipelineBuildSummary.runningCount ?: 0 + runningBuildCount = pipelineBuildSummary.runningCount ?: 0, + lastBuildFinishCount = lastBuildFinishCount, + lastBuildTotalCount = lastBuildTotalCount ) } diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt index b329ed01ce6..b822f7e635a 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt @@ -322,13 +322,19 @@ class DispatchVMStartupTaskAtom @Autowired constructor( val envId = param.thirdPartyAgentEnvId ?: "" val workspace = param.thirdPartyWorkspace ?: "" dispatchType = if (agentId.isNotBlank()) { - ThirdPartyAgentIDDispatchType(displayName = agentId, workspace = workspace, agentType = AgentType.ID) + ThirdPartyAgentIDDispatchType( + displayName = agentId, + workspace = workspace, + agentType = AgentType.ID, + dockerInfo = null + ) } else if (envId.isNotBlank()) { ThirdPartyAgentEnvDispatchType( envName = envId, envProjectId = null, workspace = workspace, - agentType = AgentType.ID + agentType = AgentType.ID, + dockerInfo = null ) } // docker建机指定版本(旧) else if (!param.dockerBuildVersion.isNullOrBlank()) { diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildCancelControl.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildCancelControl.kt index e1a5fde3b83..ad249d8b9d3 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildCancelControl.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildCancelControl.kt @@ -41,6 +41,7 @@ import com.tencent.devops.common.pipeline.utils.BuildStatusSwitcher import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.service.prometheus.BkTimed import com.tencent.devops.common.service.utils.LogUtils +import com.tencent.devops.process.engine.common.BS_CANCEL_BUILD_SOURCE import com.tencent.devops.process.engine.common.Timeout import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.process.engine.control.lock.BuildIdLock @@ -177,7 +178,7 @@ class BuildCancelControl @Autowired constructor( private fun sendBuildFinishEvent(event: PipelineBuildCancelEvent) { pipelineMQEventDispatcher.dispatch( PipelineBuildFinishEvent( - source = "cancel_build", + source = BS_CANCEL_BUILD_SOURCE, projectId = event.projectId, pipelineId = event.pipelineId, userId = event.userId, @@ -191,7 +192,7 @@ class BuildCancelControl @Autowired constructor( // #3138 buildCancel支持finallyStage pipelineMQEventDispatcher.dispatch( PipelineBuildStageEvent( - source = "cancel_build", + source = BS_CANCEL_BUILD_SOURCE, projectId = projectId, pipelineId = pipelineId, userId = event.userId, diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildStartControl.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildStartControl.kt index ca28f227ae3..46ff7b9795e 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildStartControl.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildStartControl.kt @@ -251,7 +251,7 @@ class BuildStartControl @Autowired constructor( executeCount: Int ): Boolean { var checkStart = true - val concurrencyGroup = buildInfo.concurrencyGroup ?: return true + val concurrencyGroup = buildInfo.concurrencyGroup ?: pipelineId ConcurrencyGroupLock(redisOperation, projectId, concurrencyGroup).use { groupLock -> groupLock.lock() if (buildInfo.status != BuildStatus.QUEUE_CACHE) { @@ -267,7 +267,19 @@ class BuildStartControl @Autowired constructor( projectId = projectId, concurrencyGroup = concurrencyGroup, status = listOf(BuildStatus.RUNNING) - ) + ).toMutableList() + + // #8143 兼容旧流水线版本 TODO 待模板设置补上漏洞，后期下掉 #8143 + if (concurrencyGroup == pipelineId) { + concurrencyGroupRunning.addAll( + 0, + pipelineRuntimeService.getBuildInfoListByConcurrencyGroupNull( + projectId = projectId, + pipelineId = pipelineId, + status = listOf(BuildStatus.RUNNING) + ) + ) + } LOG.info("ENGINE|$buildId|$source|CHECK_GROUP_TYPE|$concurrencyGroup|${concurrencyGroupRunning.count()}") if (concurrencyGroupRunning.isNotEmpty()) { diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/StartActionTaskContainerCmd.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/StartActionTaskContainerCmd.kt index 152e62cec22..cf6cc3bf7e8 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/StartActionTaskContainerCmd.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/StartActionTaskContainerCmd.kt @@ -202,7 +202,8 @@ class StartActionTaskContainerCmd( containerContext.buildStatus = BuildStatusSwitcher.jobStatusMaker.forceFinish(t.status, fastKill) } else { continueWhenFailure = true - if (needTerminate || t.status.isCancel()) { // #4301 强制终止的标志为失败，不管是不是设置了失败继续[P0] + if (needTerminate || t.status.isCancel() || t.status.isTerminate()) { + // #4301 强制终止的标志为失败，不管是不是设置了失败继续[P0] containerContext.buildStatus = BuildStatusSwitcher.jobStatusMaker.forceFinish(t.status) } } @@ -420,9 +421,8 @@ class StartActionTaskContainerCmd( return } // 更新task列表状态 - val startIndex = index + 1 val endIndex = containerTasks.size - 1 - for (i in startIndex..endIndex) { + for (i in index..endIndex) { val task = containerTasks[i] for (updateTaskStatusInfo in updateTaskStatusInfos) { val taskId = updateTaskStatusInfo.taskId diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/UpdateStateForStageCmdFinally.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/UpdateStateForStageCmdFinally.kt index fcced94bad0..69efaad5423 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/UpdateStateForStageCmdFinally.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/UpdateStateForStageCmdFinally.kt @@ -32,6 +32,7 @@ import com.tencent.devops.common.event.enums.ActionType import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildStatusBroadCastEvent import com.tencent.devops.common.log.utils.BuildLogPrinter import com.tencent.devops.common.pipeline.enums.BuildStatus +import com.tencent.devops.process.engine.common.BS_CANCEL_BUILD_SOURCE import com.tencent.devops.process.engine.common.BS_QUALITY_ABORT_STAGE import com.tencent.devops.process.engine.common.BS_QUALITY_PASS_STAGE import com.tencent.devops.process.engine.common.BS_STAGE_CANCELED_END_SOURCE @@ -89,7 +90,8 @@ class UpdateStateForStageCmdFinally( // Stage 暂停或者 插件暂停 if (commandContext.buildStatus == BuildStatus.STAGE_SUCCESS) { - if (event.source != BS_STAGE_CANCELED_END_SOURCE) { // 不是 stage cancel，暂停 + if (event.source != BS_STAGE_CANCELED_END_SOURCE && event.source != BS_CANCEL_BUILD_SOURCE) { + // 不是 stage cancel 或取消构建，则进行暂停逻辑 pipelineStageService.pauseStage(stage) } else { nextOrFinish(event, stage, commandContext, false) diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServiceBuildResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServiceBuildResourceImpl.kt index 44b478b9fc2..b46d06d32f5 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServiceBuildResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServiceBuildResourceImpl.kt @@ -521,6 +521,24 @@ class ServiceBuildResourceImpl @Autowired constructor( ) } + override fun getBuilds( + userId: String, + projectId: String, + pipelineId: String?, + buildStatus: Set?, + channelCode: ChannelCode + ): Result> { + return Result( + pipelineBuildFacadeService.getBuilds( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + buildStatus = buildStatus, + checkPermission = ChannelCode.isNeedAuth(channelCode) + ) + ) + } + override fun getPipelineLatestBuildByIds( projectId: String, pipelineIds: List diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt index cfce192b13e..e6013c857e8 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt @@ -147,8 +147,7 @@ class ServicePipelineResourceImpl @Autowired constructor( userId = userId, projectId = projectId, pipelineId = pipelineId, - name = pipeline.name, - desc = pipeline.desc, + pipelineCopy = pipeline, channelCode = ChannelCode.BS ) ) diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineViewResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineViewResourceImpl.kt new file mode 100644 index 00000000000..0bdae47547d --- /dev/null +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineViewResourceImpl.kt @@ -0,0 +1,191 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.api + +import com.tencent.devops.common.api.exception.CustomException +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.pipeline.enums.ChannelCode +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.process.api.service.ServicePipelineViewResource +import com.tencent.devops.process.pojo.Pipeline +import com.tencent.devops.process.pojo.PipelineCollation +import com.tencent.devops.process.pojo.PipelineSortType +import com.tencent.devops.process.pojo.classify.PipelineNewView +import com.tencent.devops.process.pojo.classify.PipelineNewViewSummary +import com.tencent.devops.process.pojo.classify.PipelineViewForm +import com.tencent.devops.process.pojo.classify.PipelineViewId +import com.tencent.devops.process.pojo.classify.PipelineViewPipelinePage +import com.tencent.devops.process.service.PipelineListFacadeService +import com.tencent.devops.process.service.view.PipelineViewGroupService +import com.tencent.devops.process.service.view.PipelineViewService +import org.springframework.beans.factory.annotation.Autowired +import javax.ws.rs.core.Response + +@RestResource +class ServicePipelineViewResourceImpl @Autowired constructor( + private val pipelineListFacadeService: PipelineListFacadeService, + private val pipelineViewService: PipelineViewService, + private val pipelineViewGroupService: PipelineViewGroupService +) : ServicePipelineViewResource { + + private fun getViewId( + viewId: String?, + viewName: String?, + projectId: String, + isProject: Boolean? + ): String { + if (!viewId.isNullOrBlank()) return viewId + if (viewName == null || isProject == null) throw CustomException( + Response.Status.BAD_REQUEST, + "和不能同时为空, 填时需同时填写参数" + ) + return pipelineViewService.viewName2viewId(projectId, viewName, isProject) + ?: throw CustomException( + Response.Status.BAD_REQUEST, + "在项目 $projectId 下未找到${if (isProject) "项目" else "个人"}视图 $viewName" + ) + } + + override fun listViewPipelines( + userId: String, + projectId: String, + page: Int?, + pageSize: Int?, + sortType: PipelineSortType?, + filterByPipelineName: String?, + filterByCreator: String?, + filterByLabels: String?, + filterByViewIds: String?, + viewId: String?, + viewName: String?, + isProject: Boolean?, + collation: PipelineCollation?, + showDelete: Boolean? + ): Result> { + return Result( + pipelineListFacadeService.listViewPipelines( + userId = userId, + projectId = projectId, + page = page, + pageSize = pageSize, + sortType = sortType ?: PipelineSortType.CREATE_TIME, + channelCode = ChannelCode.BS, + viewId = getViewId(viewId, viewName, projectId, isProject), + checkPermission = true, + filterByPipelineName = filterByPipelineName, + filterByCreator = filterByCreator, + filterByLabels = filterByLabels, + filterByViewIds = filterByViewIds, + collation = collation ?: PipelineCollation.DEFAULT, + showDelete = showDelete ?: false + ) + ) + } + + override fun listView( + userId: String, + projectId: String, + projected: Boolean?, + viewType: Int? + ): Result> { + return Result( + pipelineViewGroupService.listView( + userId = userId, + projectId = projectId, + projected = projected, + viewType = viewType + ) + ) + } + + override fun addView( + userId: String, + projectId: String, + pipelineView: PipelineViewForm + ): Result { + return Result( + PipelineViewId( + pipelineViewGroupService.addViewGroup( + projectId = projectId, + userId = userId, + pipelineView = pipelineView + ) + ) + ) + } + + override fun getView( + userId: String, + projectId: String, + viewId: String?, + viewName: String?, + isProject: Boolean? + ): Result { + return Result( + pipelineViewGroupService.getView( + userId = userId, + projectId = projectId, + viewId = getViewId(viewId, viewName, projectId, isProject) + ) + ) + } + + override fun deleteView( + userId: String, + projectId: String, + viewId: String?, + viewName: String?, + isProject: Boolean? + ): Result { + return Result( + pipelineViewGroupService.deleteViewGroup( + projectId = projectId, + userId = userId, + viewIdEncode = getViewId(viewId, viewName, projectId, isProject) + ) + ) + } + + override fun updateView( + userId: String, + projectId: String, + viewId: String?, + viewName: String?, + isProject: Boolean?, + pipelineView: PipelineViewForm + ): Result { + return Result( + pipelineViewGroupService.updateViewGroup( + projectId = projectId, + userId = userId, + viewIdEncode = getViewId(viewId, viewName, projectId, isProject), + pipelineView = pipelineView + ) + ) + } +} diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserBuildResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserBuildResourceImpl.kt index 6f7c4d965c8..01f7626fc03 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserBuildResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserBuildResourceImpl.kt @@ -45,6 +45,7 @@ import com.tencent.devops.process.pojo.BuildId import com.tencent.devops.process.pojo.BuildManualStartupInfo import com.tencent.devops.process.pojo.ReviewParam import com.tencent.devops.process.pojo.pipeline.ModelDetail +import com.tencent.devops.process.service.PipelineRecentUseService import com.tencent.devops.process.service.builds.PipelineBuildFacadeService import com.tencent.devops.process.service.builds.PipelinePauseBuildFacadeService import io.micrometer.core.annotation.Timed @@ -55,7 +56,8 @@ import javax.ws.rs.core.Response @Suppress("ALL") class UserBuildResourceImpl @Autowired constructor( private val pipelineBuildFacadeService: PipelineBuildFacadeService, - private val pipelinePauseBuildFacadeService: PipelinePauseBuildFacadeService + private val pipelinePauseBuildFacadeService: PipelinePauseBuildFacadeService, + private val pipelineRecentUseService: PipelineRecentUseService ) : UserBuildResource { override fun manualStartupInfo( @@ -89,20 +91,18 @@ class UserBuildResourceImpl @Autowired constructor( triggerReviewers: List? ): Result { checkParam(userId, projectId, pipelineId) - return Result( - BuildId( - pipelineBuildFacadeService.buildManualStartup( - userId = userId, - startType = StartType.MANUAL, - projectId = projectId, - pipelineId = pipelineId, - values = values, - channelCode = ChannelCode.BS, - buildNo = buildNo, - triggerReviewers = triggerReviewers - ) - ) + val manualStartup = pipelineBuildFacadeService.buildManualStartup( + userId = userId, + startType = StartType.MANUAL, + projectId = projectId, + pipelineId = pipelineId, + values = values, + channelCode = ChannelCode.BS, + buildNo = buildNo, + triggerReviewers = triggerReviewers ) + pipelineRecentUseService.record(userId, projectId, pipelineId) + return Result(BuildId(manualStartup)) } override fun retry( @@ -255,15 +255,15 @@ class UserBuildResourceImpl @Autowired constructor( if (buildId.isBlank()) { throw ParamBlankException("Invalid buildId") } - return Result( - pipelineBuildFacadeService.getBuildDetail( - userId = userId, - projectId = projectId, - pipelineId = pipelineId, - buildId = buildId, - channelCode = ChannelCode.BS - ) + val buildDetail = pipelineBuildFacadeService.getBuildDetail( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + channelCode = ChannelCode.BS ) + pipelineRecentUseService.record(userId, projectId, pipelineId) + return Result(buildDetail) } override fun getBuildDetailByBuildNo( @@ -374,6 +374,7 @@ class UserBuildResourceImpl @Autowired constructor( buildNoEnd = buildNoEnd, buildMsg = buildMsg ) + pipelineRecentUseService.record(userId, projectId, pipelineId) return Result(result) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt index e5b37faae95..d0504cdc877 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt @@ -28,6 +28,7 @@ package com.tencent.devops.process.api import com.tencent.devops.common.api.exception.ErrorCodeException +import com.tencent.devops.common.api.exception.InvalidParamException import com.tencent.devops.common.api.exception.ParamBlankException import com.tencent.devops.common.api.pojo.Page import com.tencent.devops.common.api.pojo.Result @@ -35,6 +36,8 @@ import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.common.auth.api.AuthResourceType import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.enums.ChannelCode +import com.tencent.devops.common.pipeline.pojo.MatrixPipelineInfo +import com.tencent.devops.common.pipeline.utils.MatrixYamlCheckUtils import com.tencent.devops.common.web.RestResource import com.tencent.devops.process.api.user.UserPipelineResource import com.tencent.devops.process.audit.service.AuditService @@ -43,9 +46,9 @@ import com.tencent.devops.process.engine.pojo.PipelineInfo import com.tencent.devops.process.engine.service.PipelineVersionFacadeService import com.tencent.devops.process.engine.service.rule.PipelineRuleService import com.tencent.devops.process.permission.PipelinePermissionService -import com.tencent.devops.common.pipeline.pojo.MatrixPipelineInfo import com.tencent.devops.process.pojo.Permission import com.tencent.devops.process.pojo.Pipeline +import com.tencent.devops.process.pojo.PipelineCollation import com.tencent.devops.process.pojo.PipelineCopy import com.tencent.devops.process.pojo.PipelineId import com.tencent.devops.process.pojo.PipelineName @@ -57,16 +60,18 @@ import com.tencent.devops.process.pojo.app.PipelinePage import com.tencent.devops.process.pojo.audit.Audit import com.tencent.devops.process.pojo.classify.PipelineViewAndPipelines import com.tencent.devops.process.pojo.classify.PipelineViewPipelinePage +import com.tencent.devops.process.pojo.pipeline.BatchDeletePipeline +import com.tencent.devops.process.pojo.pipeline.PipelineCount import com.tencent.devops.process.pojo.pipeline.enums.PipelineRuleBusCodeEnum import com.tencent.devops.process.pojo.setting.PipelineModelAndSetting import com.tencent.devops.process.pojo.setting.PipelineSetting import com.tencent.devops.process.service.PipelineInfoFacadeService import com.tencent.devops.process.service.PipelineListFacadeService +import com.tencent.devops.process.service.PipelineRecentUseService import com.tencent.devops.process.service.PipelineRemoteAuthService import com.tencent.devops.process.service.StageTagService import com.tencent.devops.process.service.label.PipelineGroupService import com.tencent.devops.process.service.pipeline.PipelineSettingFacadeService -import com.tencent.devops.common.pipeline.utils.MatrixYamlCheckUtils import io.micrometer.core.annotation.Timed import org.springframework.beans.factory.annotation.Autowired import javax.ws.rs.core.Response @@ -82,7 +87,8 @@ class UserPipelineResourceImpl @Autowired constructor( private val pipelineInfoFacadeService: PipelineInfoFacadeService, private val auditService: AuditService, private val pipelineVersionFacadeService: PipelineVersionFacadeService, - private val pipelineRuleService: PipelineRuleService + private val pipelineRuleService: PipelineRuleService, + private val pipelineRecentUseService: PipelineRecentUseService ) : UserPipelineResource { override fun hasCreatePermission(userId: String, projectId: String): Result { @@ -213,8 +219,7 @@ class UserPipelineResourceImpl @Autowired constructor( userId = userId, projectId = projectId, pipelineId = pipelineId, - name = pipeline.name, - desc = pipeline.desc, + pipelineCopy = pipeline, channelCode = ChannelCode.BS ) ) @@ -336,14 +341,14 @@ class UserPipelineResourceImpl @Autowired constructor( override fun get(userId: String, projectId: String, pipelineId: String): Result { checkParam(userId, projectId) - return Result( - pipelineInfoFacadeService.getPipeline( - userId = userId, - projectId = projectId, - pipelineId = pipelineId, - channelCode = ChannelCode.BS - ) + val pipeline = pipelineInfoFacadeService.getPipeline( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + channelCode = ChannelCode.BS ) + pipelineRecentUseService.record(userId, projectId, pipelineId) + return Result(pipeline) } override fun getVersion(userId: String, projectId: String, pipelineId: String, version: Int): Result { @@ -403,6 +408,24 @@ class UserPipelineResourceImpl @Autowired constructor( return Result(true) } + override fun batchDelete(userId: String, batchDeletePipeline: BatchDeletePipeline): Result> { + val pipelineIds = batchDeletePipeline.pipelineIds + if (pipelineIds.isEmpty()) { + return Result(emptyMap()) + } + if (pipelineIds.size > 100) { + throw InvalidParamException(message = "流水线列表长度不能超过100") + } + val result = pipelineIds.associateWith { + try { + softDelete(userId, batchDeletePipeline.projectId, it).data ?: false + } catch (e: Exception) { + false + } + } + return Result(result) + } + override fun deleteVersion( userId: String, projectId: String, @@ -430,6 +453,11 @@ class UserPipelineResourceImpl @Autowired constructor( return Result(true) } + override fun getCount(userId: String, projectId: String): Result { + checkParam(userId, projectId) + return Result(pipelineListFacadeService.getCount(userId, projectId)) + } + override fun restore(userId: String, projectId: String, pipelineId: String): Result { checkParam(userId, projectId) pipelineInfoFacadeService.restorePipeline( @@ -446,13 +474,18 @@ class UserPipelineResourceImpl @Autowired constructor( projectId: String, page: Int?, pageSize: Int?, - sortType: PipelineSortType? + sortType: PipelineSortType?, + collation: PipelineCollation? ): Result> { checkParam(userId, projectId) return Result( pipelineListFacadeService.listDeletePipelineIdByProject( - userId, projectId, page, - pageSize, sortType ?: PipelineSortType.CREATE_TIME, ChannelCode.BS + userId = userId, + projectId = projectId, + page = page, + pageSize = pageSize, + sortType = sortType ?: PipelineSortType.CREATE_TIME, ChannelCode.BS, + collation = collation ?: PipelineCollation.DEFAULT ) ) } @@ -477,7 +510,10 @@ class UserPipelineResourceImpl @Autowired constructor( filterByPipelineName: String?, filterByCreator: String?, filterByLabels: String?, - viewId: String + filterByViewIds: String?, + viewId: String, + collation: PipelineCollation?, + showDelete: Boolean? ): Result> { checkParam(userId, projectId) return Result( @@ -492,7 +528,10 @@ class UserPipelineResourceImpl @Autowired constructor( checkPermission = true, filterByPipelineName = filterByPipelineName, filterByCreator = filterByCreator, - filterByLabels = filterByLabels + filterByLabels = filterByLabels, + filterByViewIds = filterByViewIds, + collation = collation ?: PipelineCollation.DEFAULT, + showDelete = showDelete ?: false ) ) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineViewResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineViewResourceImpl.kt index cfc6c5e3f85..98edb4f846c 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineViewResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineViewResourceImpl.kt @@ -31,17 +31,27 @@ import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.web.RestResource import com.tencent.devops.process.api.user.UserPipelineViewResource import com.tencent.devops.process.pojo.classify.PipelineNewView -import com.tencent.devops.process.pojo.classify.PipelineNewViewCreate import com.tencent.devops.process.pojo.classify.PipelineNewViewSummary -import com.tencent.devops.process.pojo.classify.PipelineNewViewUpdate +import com.tencent.devops.process.pojo.classify.PipelineViewBulkAdd +import com.tencent.devops.process.pojo.classify.PipelineViewBulkRemove +import com.tencent.devops.process.pojo.classify.PipelineViewDict +import com.tencent.devops.process.pojo.classify.PipelineViewForm +import com.tencent.devops.process.pojo.classify.PipelineViewHitFilters import com.tencent.devops.process.pojo.classify.PipelineViewId +import com.tencent.devops.process.pojo.classify.PipelineViewMatchDynamic +import com.tencent.devops.process.pojo.classify.PipelineViewPipelineCount +import com.tencent.devops.process.pojo.classify.PipelineViewPreview import com.tencent.devops.process.pojo.classify.PipelineViewSettings +import com.tencent.devops.process.pojo.classify.PipelineViewTopForm +import com.tencent.devops.process.service.view.PipelineViewGroupService import com.tencent.devops.process.service.view.PipelineViewService import org.springframework.beans.factory.annotation.Autowired @RestResource -class UserPipelineViewResourceImpl @Autowired constructor(private val pipelineViewService: PipelineViewService) : - UserPipelineViewResource { +class UserPipelineViewResourceImpl @Autowired constructor( + private val pipelineViewService: PipelineViewService, + private val pipelineViewGroupService: PipelineViewGroupService +) : UserPipelineViewResource { override fun getViewSettings(userId: String, projectId: String): Result { return Result(pipelineViewService.getViewSettings(userId, projectId)) } @@ -56,27 +66,94 @@ class UserPipelineViewResourceImpl @Autowired constructor(private val pipelineVi } override fun getView(userId: String, projectId: String, viewId: String): Result { - return Result(pipelineViewService.getView(userId, projectId, viewId)) + return Result(pipelineViewGroupService.getView(userId, projectId, viewId)) } override fun addView( userId: String, projectId: String, - pipelineView: PipelineNewViewCreate + pipelineView: PipelineViewForm ): Result { - return Result(PipelineViewId(pipelineViewService.addView(userId, projectId, pipelineView))) + return Result(PipelineViewId(pipelineViewGroupService.addViewGroup(projectId, userId, pipelineView))) + } + + override fun listViewByPipelineId( + userId: String, + projectId: String, + pipelineId: String + ): Result> { + return Result(pipelineViewGroupService.listViewByPipelineId(userId, projectId, pipelineId)) + } + + override fun topView( + userId: String, + projectId: String, + viewId: String, + pipelineViewTopForm: PipelineViewTopForm + ): Result { + return Result(pipelineViewService.topView(userId, projectId, viewId, pipelineViewTopForm.enabled)) + } + + override fun preview( + userId: String, + projectId: String, + pipelineView: PipelineViewForm + ): Result { + return Result(pipelineViewGroupService.preview(userId, projectId, pipelineView)) + } + + override fun dict(userId: String, projectId: String): Result { + return Result(pipelineViewGroupService.dict(userId, projectId)) + } + + override fun getHitFilters( + userId: String, + projectId: String, + pipelineId: String, + viewId: String + ): Result { + return Result(pipelineViewService.getHitFilters(userId, projectId, pipelineId, viewId)) + } + + override fun matchDynamicView( + userId: String, + projectId: String, + pipelineViewMatchDynamic: PipelineViewMatchDynamic + ): Result> { + return Result(pipelineViewService.matchDynamicView(userId, projectId, pipelineViewMatchDynamic)) + } + + override fun listView( + userId: String, + projectId: String, + projected: Boolean?, + viewType: Int? + ): Result> { + return Result(pipelineViewGroupService.listView(userId, projectId, projected, viewType)) + } + + override fun pipelineCount(userId: String, projectId: String, viewId: String): Result { + return Result(pipelineViewGroupService.pipelineCount(userId, projectId, viewId)) + } + + override fun bulkAdd(userId: String, projectId: String, bulkAdd: PipelineViewBulkAdd): Result { + return Result(pipelineViewGroupService.bulkAdd(userId, projectId, bulkAdd)) + } + + override fun bulkRemove(userId: String, projectId: String, bulkRemove: PipelineViewBulkRemove): Result { + return Result(pipelineViewGroupService.bulkRemove(userId, projectId, bulkRemove)) } override fun deleteView(userId: String, projectId: String, viewId: String): Result { - return Result(pipelineViewService.deleteView(userId, projectId, viewId)) + return Result(pipelineViewGroupService.deleteViewGroup(projectId, userId, viewId)) } override fun updateView( userId: String, projectId: String, viewId: String, - pipelineView: PipelineNewViewUpdate + pipelineView: PipelineViewForm ): Result { - return Result(pipelineViewService.updateView(userId, projectId, viewId, pipelineView)) + return Result(pipelineViewGroupService.updateViewGroup(projectId, userId, viewId, pipelineView)) } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserSubPipelineInfoResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserSubPipelineInfoResourceImpl.kt new file mode 100644 index 00000000000..70740925d1f --- /dev/null +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserSubPipelineInfoResourceImpl.kt @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.api + +import com.tencent.devops.common.api.exception.ParamBlankException +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.process.api.user.UserSubPipelineInfoResource +import com.tencent.devops.process.pojo.pipeline.SubPipelineStartUpInfo +import com.tencent.devops.process.service.SubPipelineStartUpService +import org.springframework.beans.factory.annotation.Autowired + +@RestResource +class UserSubPipelineInfoResourceImpl @Autowired constructor ( + private val subPipeService: SubPipelineStartUpService +) : UserSubPipelineInfoResource { + + override fun subpipManualStartupInfo( + userId: String, + projectId: String, + pipelineId: String + ): Result> { + checkParam(userId) + return subPipeService.subPipelineManualStartupInfo( + userId = userId, projectId = projectId, pipelineId = pipelineId + ) + } + + private fun checkParam(userId: String) { + if (userId.isBlank()) { + throw ParamBlankException("Invalid userId") + } + } +} diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineViewResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineViewResourceImpl.kt new file mode 100644 index 00000000000..320f74f8512 --- /dev/null +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineViewResourceImpl.kt @@ -0,0 +1,16 @@ +package com.tencent.devops.process.api.op + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.process.service.view.PipelineViewGroupService +import org.springframework.beans.factory.annotation.Autowired + +@RestResource +class OpPipelineViewResourceImpl @Autowired constructor( + private val pipelineViewGroupService: PipelineViewGroupService +) : OpPipelineViewResource { + override fun initAllView(userId: String): Result { + Thread { pipelineViewGroupService.initAllView() }.start() + return Result(true) + } +} diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/template/ServiceTemplateInstanceResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/template/ServiceTemplateInstanceResourceImpl.kt index 5eb8dcedc6f..6fd007d4b30 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/template/ServiceTemplateInstanceResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/template/ServiceTemplateInstanceResourceImpl.kt @@ -64,7 +64,7 @@ class ServiceTemplateInstanceResourceImpl @Autowired constructor( } override fun countTemplateInstance(projectId: String, templateIds: Collection): Result { - return Result(templateFacadeService.serviceCountTemplateInstances(projectId, templateIds)) + return Result(data = templateFacadeService.serviceCountTemplateInstances(projectId, templateIds)) } override fun countTemplateInstanceDetail( diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineCreateListener.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineCreateListener.kt index 6ff77f43d97..60fba26f97b 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineCreateListener.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineCreateListener.kt @@ -56,34 +56,34 @@ class MQPipelineCreateListener @Autowired constructor( override fun run(event: PipelineCreateEvent) { val watcher = Watcher(id = "${event.traceId}|CreatePipeline#${event.pipelineId}|${event.userId}") - try { - watcher.start("callback") + watcher.safeAround("callback") { callBackControl.pipelineCreateEvent(projectId = event.projectId, pipelineId = event.pipelineId) - watcher.stop() - watcher.start("updateAtomPipelineNum") + } + + watcher.safeAround("updateAtomPipelineNum") { pipelineAtomStatisticsService.updateAtomPipelineNum( projectId = event.projectId, pipelineId = event.pipelineId, version = event.version ?: 1 ) - watcher.stop() - watcher.start("updateAgentPipelineRef") + } + + watcher.safeAround("updateAgentPipelineRef") { with(event) { agentPipelineRefService.updateAgentPipelineRef(userId, "create_pipeline", projectId, pipelineId) } - watcher.stop() - watcher.start("addWebhook") + } + + watcher.safeAround("addWebhook") { pipelineWebhookService.addWebhook( projectId = event.projectId, pipelineId = event.pipelineId, version = event.version, userId = event.userId ) - watcher.stop() - } finally { - watcher.stop() - LogUtils.printCostTimeWE(watcher = watcher) } + + LogUtils.printCostTimeWE(watcher = watcher) } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineDeleteListener.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineDeleteListener.kt index 67789b7cf02..e9ff55ff847 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineDeleteListener.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineDeleteListener.kt @@ -60,38 +60,41 @@ class MQPipelineDeleteListener @Autowired constructor( override fun run(event: PipelineDeleteEvent) { val watcher = Watcher(id = "${event.traceId}|DeletePipeline#${event.pipelineId}|${event.clearUpModel}") - try { - val projectId = event.projectId - val pipelineId = event.pipelineId - val userId = event.userId + val projectId = event.projectId + val pipelineId = event.pipelineId + val userId = event.userId - watcher.start("cancelPendingTask") + watcher.safeAround("cancelPendingTask") { pipelineRuntimeService.cancelPendingTask(projectId, pipelineId, userId) - watcher.stop() + } - if (event.clearUpModel) { - watcher.start("deleteExt") + if (event.clearUpModel) { + watcher.safeAround("deleteExt") { pipelineGroupService.deleteAllUserFavorByPipeline(userId, projectId, pipelineId) // 删除收藏该流水线上所有记录 pipelineGroupService.deletePipelineLabel(userId, projectId, pipelineId) } - watcher.start("deleteWebhook") + } + + watcher.safeAround("deleteWebhook") { pipelineWebhookService.deleteWebhook(projectId, pipelineId, userId) - watcher.stop() - watcher.start("updateAgentPipelineRef") + } + + watcher.safeAround("updateAgentPipelineRef") { agentPipelineRefService.updateAgentPipelineRef(userId, "delete_pipeline", projectId, pipelineId) - watcher.stop() - watcher.start("updateAtomPipelineNum") + } + + watcher.safeAround("updateAtomPipelineNum") { pipelineAtomStatisticsService.updateAtomPipelineNum( projectId = event.projectId, pipelineId = event.pipelineId, deleteFlag = true ) - watcher.stop() - watcher.start("callback") + } + + watcher.safeAround("callback") { callBackControl.pipelineDeleteEvent(projectId = event.projectId, pipelineId = event.pipelineId) - } finally { - watcher.stop() - LogUtils.printCostTimeWE(watcher = watcher) } + + LogUtils.printCostTimeWE(watcher = watcher) } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineUpdateListener.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineUpdateListener.kt index 13940552abb..d1fe14c9335 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineUpdateListener.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/listener/pipeline/MQPipelineUpdateListener.kt @@ -51,40 +51,42 @@ class MQPipelineUpdateListener @Autowired constructor( private val pipelineAtomStatisticsService: PipelineAtomStatisticsService, private val callBackControl: CallBackControl, private val agentPipelineRefService: AgentPipelineRefService, - pipelineEventDispatcher: PipelineEventDispatcher, - private val pipelineWebhookService: PipelineWebhookService + private val pipelineWebhookService: PipelineWebhookService, + pipelineEventDispatcher: PipelineEventDispatcher ) : BaseListener(pipelineEventDispatcher) { override fun run(event: PipelineUpdateEvent) { val watcher = Watcher(id = "${event.traceId}|UpdatePipeline#${event.pipelineId}|${event.userId}") - try { - if (event.buildNo != null) { - watcher.start("updateBuildNo") + + if (event.buildNo != null) { + watcher.safeAround("updateBuildNo") { pipelineRuntimeService.updateBuildNo(event.projectId, event.pipelineId, event.buildNo!!.buildNo) - watcher.stop() } - watcher.start("callback") + } + + watcher.safeAround("callback") { callBackControl.pipelineUpdateEvent(projectId = event.projectId, pipelineId = event.pipelineId) - watcher.stop() - watcher.start("updateAgentPipelineRef") + } + + watcher.safeAround("updateAgentPipelineRef") { with(event) { agentPipelineRefService.updateAgentPipelineRef(userId, "update_pipeline", projectId, pipelineId) } - watcher.stop() - watcher.start("updateAtomPipelineNum") + } + + watcher.safeAround("updateAtomPipelineNum") { pipelineAtomStatisticsService.updateAtomPipelineNum(event.projectId, event.pipelineId, event.version) - watcher.stop() - watcher.start("addWebhook") + } + + watcher.safeAround("addWebhook") { pipelineWebhookService.addWebhook( projectId = event.projectId, pipelineId = event.pipelineId, version = event.version, userId = event.userId ) - watcher.stop() - } finally { - watcher.stop() - LogUtils.printCostTimeWE(watcher) } + + LogUtils.printCostTimeWE(watcher) } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/service/template/TemplateInstanceCronService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/service/template/TemplateInstanceCronService.kt index 29f645f1f8e..e1530e30231 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/service/template/TemplateInstanceCronService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/service/template/TemplateInstanceCronService.kt @@ -74,8 +74,8 @@ class TemplateInstanceCronService @Autowired constructor( @Value("\${template.instanceListUrl}") private val instanceListUrl: String = "" - @Value("\${template.maxErrorReasonLength:400}") - private val maxErrorReasonLength: Int = 400 + @Value("\${template.maxErrorReasonLength:200}") + private val maxErrorReasonLength: Int = 200 @Scheduled(cron = "0 0/1 * * * ?") fun templateInstance() { diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt index a73d29f7b32..f5bd65a4dc9 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt @@ -63,6 +63,8 @@ import com.tencent.devops.process.engine.utils.PipelineUtils import com.tencent.devops.process.jmx.api.ProcessJmxApi import com.tencent.devops.process.jmx.pipeline.PipelineBean import com.tencent.devops.process.permission.PipelinePermissionService +import com.tencent.devops.process.pojo.PipelineCopy +import com.tencent.devops.process.pojo.classify.PipelineViewBulkAdd import com.tencent.devops.process.pojo.pipeline.DeletePipelineResult import com.tencent.devops.process.pojo.pipeline.DeployPipelineResult import com.tencent.devops.process.pojo.setting.PipelineModelAndSetting @@ -70,6 +72,7 @@ import com.tencent.devops.process.pojo.setting.PipelineSetting import com.tencent.devops.process.pojo.template.TemplateType import com.tencent.devops.process.service.label.PipelineGroupService import com.tencent.devops.process.service.pipeline.PipelineSettingFacadeService +import com.tencent.devops.process.service.view.PipelineViewGroupService import com.tencent.devops.process.template.service.TemplateService import com.tencent.devops.store.api.template.ServiceTemplateResource import org.jooq.DSLContext @@ -93,6 +96,7 @@ class PipelineInfoFacadeService @Autowired constructor( private val pipelineSettingFacadeService: PipelineSettingFacadeService, private val pipelineRepositoryService: PipelineRepositoryService, private val pipelineGroupService: PipelineGroupService, + private val pipelineViewGroupService: PipelineViewGroupService, private val pipelinePermissionService: PipelinePermissionService, private val stageTagService: StageTagService, private val templateService: TemplateService, @@ -101,7 +105,8 @@ class PipelineInfoFacadeService @Autowired constructor( private val processJmxApi: ProcessJmxApi, private val client: Client, private val pipelineInfoDao: PipelineInfoDao, - private val redisOperation: RedisOperation + private val redisOperation: RedisOperation, + private val pipelineRecentUseService: PipelineRecentUseService ) { @Value("\${process.deletedPipelineStoreDays:30}") @@ -365,6 +370,8 @@ class PipelineInfoFacadeService @Autowired constructor( } watcher.stop() } + + // 添加标签 pipelineGroupService.addPipelineLabel( userId = userId, projectId = projectId, @@ -372,6 +379,17 @@ class PipelineInfoFacadeService @Autowired constructor( labelIds = model.labels ) + // 添加到静态分组 + val bulkAdd = PipelineViewBulkAdd(pipelineIds = listOf(pipelineId), viewIds = model.staticViews) + pipelineViewGroupService.bulkAdd(userId, projectId, bulkAdd) + + // 添加到动态分组 + pipelineViewGroupService.updateGroupAfterPipelineCreate( + projectId = projectId, + pipelineId = pipelineId, + userId = userId + ) + success = true return pipelineId } catch (duplicateKeyException: DuplicateKeyException) { @@ -460,12 +478,10 @@ class PipelineInfoFacadeService @Autowired constructor( userId: String, projectId: String, pipelineId: String, - name: String, - desc: String?, + pipelineCopy: PipelineCopy, channelCode: ChannelCode, checkPermission: Boolean = true ): String { - val pipeline = pipelineRepositoryService.getPipelineInfo(projectId, pipelineId) ?: throw ErrorCodeException( statusCode = Response.Status.NOT_FOUND.statusCode, @@ -515,7 +531,13 @@ class PipelineInfoFacadeService @Autowired constructor( defaultMessage = "指定要复制的流水线-模型不存在" ) try { - val copyMode = Model(name, desc ?: model.desc, model.stages) + val copyMode = Model( + name = pipelineCopy.name, + desc = pipelineCopy.desc ?: model.desc, + stages = model.stages, + staticViews = pipelineCopy.staticViews, + labels = pipelineCopy.labels + ) modelCheckPlugin.clearUpModel(copyMode) val newPipelineId = createPipeline(userId, projectId, copyMode, channelCode) val settingInfo = pipelineSettingFacadeService.getSettingInfo(projectId, pipelineId) @@ -525,13 +547,14 @@ class PipelineInfoFacadeService @Autowired constructor( oldSetting = settingInfo, projectId = projectId, newPipelineId = newPipelineId, - pipelineName = name + pipelineName = pipelineCopy.name ) // 复制setting到新流水线 pipelineSettingFacadeService.saveSetting( userId = userId, setting = newSetting, - dispatchPipelineUpdateEvent = false + dispatchPipelineUpdateEvent = false, + updateLabels = false ) } return newPipelineId @@ -714,7 +737,6 @@ class PipelineInfoFacadeService @Autowired constructor( version: Int? = null, checkPermission: Boolean = true ): Model { - if (checkPermission) { pipelinePermissionService.validPipelinePermission( userId = userId, @@ -808,8 +830,11 @@ class PipelineInfoFacadeService @Autowired constructor( if (checkPermission) { watcher.start("perm_v_perm") pipelinePermissionService.validPipelinePermission( - userId = userId, projectId = projectId, pipelineId = pipelineId, - permission = AuthPermission.DELETE, message = "用户($userId)无权限在工程($projectId)下删除流水线($pipelineId)" + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + permission = AuthPermission.DELETE, + message = "用户($userId)无权限在工程($projectId)下删除流水线($pipelineId)" ) watcher.stop() } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineListFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineListFacadeService.kt index 0e4a82c1f47..5f3ce926e92 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineListFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineListFacadeService.kt @@ -27,6 +27,7 @@ package com.tencent.devops.process.service +import com.fasterxml.jackson.core.type.TypeReference import com.tencent.devops.common.api.constant.CommonMessageCode import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.exception.ParamBlankException @@ -34,27 +35,32 @@ import com.tencent.devops.common.api.exception.PermissionForbiddenException import com.tencent.devops.common.api.model.SQLLimit import com.tencent.devops.common.api.model.SQLPage import com.tencent.devops.common.api.pojo.Page -import com.tencent.devops.common.event.pojo.measure.PipelineLabelRelateInfo import com.tencent.devops.common.api.util.DateTimeUtil +import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.PageUtil import com.tencent.devops.common.api.util.Watcher import com.tencent.devops.common.api.util.timestampmilli import com.tencent.devops.common.auth.api.AuthPermission +import com.tencent.devops.common.event.pojo.measure.PipelineLabelRelateInfo import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.enums.PipelineInstanceTypeEnum +import com.tencent.devops.common.pipeline.pojo.element.trigger.enums.CodeEventType import com.tencent.devops.common.service.utils.LogUtils import com.tencent.devops.common.service.utils.MessageCodeUtil import com.tencent.devops.model.process.tables.TPipelineSetting import com.tencent.devops.model.process.tables.TTemplatePipeline +import com.tencent.devops.model.process.tables.records.TPipelineBuildHistoryRecord import com.tencent.devops.model.process.tables.records.TPipelineBuildSummaryRecord import com.tencent.devops.model.process.tables.records.TPipelineInfoRecord import com.tencent.devops.process.constant.ProcessMessageCode import com.tencent.devops.process.dao.PipelineFavorDao import com.tencent.devops.process.dao.PipelineSettingDao import com.tencent.devops.process.dao.label.PipelineLabelPipelineDao +import com.tencent.devops.process.engine.dao.PipelineBuildDao import com.tencent.devops.process.engine.dao.PipelineBuildSummaryDao +import com.tencent.devops.process.engine.dao.PipelineBuildTaskDao import com.tencent.devops.process.engine.dao.PipelineInfoDao import com.tencent.devops.process.engine.dao.template.TemplatePipelineDao import com.tencent.devops.process.engine.pojo.PipelineFilterByLabelInfo @@ -65,6 +71,7 @@ import com.tencent.devops.process.engine.service.PipelineRuntimeService import com.tencent.devops.process.jmx.api.ProcessJmxApi import com.tencent.devops.process.permission.PipelinePermissionService import com.tencent.devops.process.pojo.Pipeline +import com.tencent.devops.process.pojo.PipelineCollation import com.tencent.devops.process.pojo.PipelineDetailInfo import com.tencent.devops.process.pojo.PipelineIdAndName import com.tencent.devops.process.pojo.PipelineIdInfo @@ -78,17 +85,24 @@ import com.tencent.devops.process.pojo.classify.PipelineViewFilterByName import com.tencent.devops.process.pojo.classify.PipelineViewPipelinePage import com.tencent.devops.process.pojo.classify.enums.Condition import com.tencent.devops.process.pojo.classify.enums.Logic +import com.tencent.devops.process.pojo.code.WebhookInfo +import com.tencent.devops.process.pojo.pipeline.PipelineCount import com.tencent.devops.process.pojo.pipeline.SimplePipeline import com.tencent.devops.process.pojo.setting.PipelineRunLockType import com.tencent.devops.process.pojo.template.TemplatePipelineInfo import com.tencent.devops.process.service.label.PipelineGroupService import com.tencent.devops.process.service.pipeline.PipelineStatusService +import com.tencent.devops.process.service.view.PipelineViewGroupService import com.tencent.devops.process.service.view.PipelineViewService import com.tencent.devops.process.utils.KEY_PIPELINE_ID import com.tencent.devops.process.utils.PIPELINE_VIEW_ALL_PIPELINES import com.tencent.devops.process.utils.PIPELINE_VIEW_FAVORITE_PIPELINES +import com.tencent.devops.process.utils.PIPELINE_VIEW_MY_LIST_PIPELINES import com.tencent.devops.process.utils.PIPELINE_VIEW_MY_PIPELINES +import com.tencent.devops.process.utils.PIPELINE_VIEW_RECENT_USE +import com.tencent.devops.process.utils.PIPELINE_VIEW_UNCLASSIFIED import com.tencent.devops.quality.api.v2.pojo.response.QualityPipeline +import com.tencent.devops.scm.utils.code.git.GitUtils import org.jooq.DSLContext import org.jooq.Record4 import org.jooq.Result @@ -107,6 +121,7 @@ class PipelineListFacadeService @Autowired constructor( private val pipelinePermissionService: PipelinePermissionService, private val pipelineGroupService: PipelineGroupService, private val pipelineViewService: PipelineViewService, + private val pipelineViewGroupService: PipelineViewGroupService, private val pipelineStatusService: PipelineStatusService, private val processJmxApi: ProcessJmxApi, private val dslContext: DSLContext, @@ -114,8 +129,11 @@ class PipelineListFacadeService @Autowired constructor( private val pipelineInfoDao: PipelineInfoDao, private val pipelineSettingDao: PipelineSettingDao, private val pipelineBuildSummaryDao: PipelineBuildSummaryDao, + private val pipelineBuildDao: PipelineBuildDao, + private val pipelineBuildTaskDao: PipelineBuildTaskDao, private val pipelineFavorDao: PipelineFavorDao, - private val pipelineLabelPipelineDao: PipelineLabelPipelineDao + private val pipelineLabelPipelineDao: PipelineLabelPipelineDao, + private val pipelineRecentUseService: PipelineRecentUseService ) { @Value("\${process.deletedPipelineStoreDays:30}") @@ -134,12 +152,15 @@ class PipelineListFacadeService @Autowired constructor( PipelineSortType.NAME -> { a.pipelineName.toLowerCase().compareTo(b.pipelineName.toLowerCase()) } + PipelineSortType.CREATE_TIME -> { b.createTime.compareTo(a.createTime) } + PipelineSortType.UPDATE_TIME -> { b.deploymentTime.compareTo(a.deploymentTime) } + PipelineSortType.LAST_EXEC_TIME -> { b.deploymentTime.compareTo(a.latestBuildStartTime ?: 0) } @@ -179,7 +200,8 @@ class PipelineListFacadeService @Autowired constructor( projectId = projectId, channelCode = channelCode, sortType = null, - pipelineIds = pipelineIds + pipelineIds = pipelineIds, + userId = userId ) return buildPipelines( @@ -233,7 +255,8 @@ class PipelineListFacadeService @Autowired constructor( val buildPipelineRecords = pipelineRuntimeService.getBuildPipelineRecords( projectId = projectId, channelCode = ChannelCode.BS, - pipelineIds = resultPipelineIds) + pipelineIds = resultPipelineIds + ) if (buildPipelineRecords.isNotEmpty) { pipelines.addAll( buildPipelines( @@ -309,7 +332,8 @@ class PipelineListFacadeService @Autowired constructor( dslContext = dslContext, projectId = projectId, channelCode = channelCode, - pipelineIds = hasPermissionList + pipelineIds = hasPermissionList, + userId = userId ) watcher.stop() @@ -417,18 +441,15 @@ class PipelineListFacadeService @Autowired constructor( filterByCreator: String? = null, filterByLabels: String? = null, filterInvalid: Boolean = false, - authPipelineIds: List = emptyList(), - skipPipelineIds: List = emptyList() + filterByViewIds: String? = null, + collation: PipelineCollation = PipelineCollation.DEFAULT, + showDelete: Boolean = false ): PipelineViewPipelinePage { val watcher = Watcher(id = "listViewPipelines|$projectId|$userId") watcher.start("perm_r_perm") - val authPipelines = if (authPipelineIds.isEmpty()) { - pipelinePermissionService.getResourceByPermission( - userId = userId, projectId = projectId, permission = AuthPermission.LIST - ) - } else { - authPipelineIds - } + val authPipelines = pipelinePermissionService.getResourceByPermission( + userId = userId, projectId = projectId, permission = AuthPermission.LIST + ) watcher.stop() watcher.start("s_r_summary") @@ -455,22 +476,43 @@ class PipelineListFacadeService @Autowired constructor( ) ) pipelineFilterParamList.add(pipelineFilterParam) - val viewIdList = - listOf(PIPELINE_VIEW_FAVORITE_PIPELINES, PIPELINE_VIEW_MY_PIPELINES, PIPELINE_VIEW_ALL_PIPELINES) - if (!viewIdList.contains(viewId)) { - val view = pipelineViewService.getView(userId = userId, projectId = projectId, viewId = viewId) - val filters = pipelineViewService.getFilters(view) - val pipelineViewFilterParam = PipelineFilterParam( - logic = view.logic, - filterByPipelineNames = filters.first, - filterByPipelineCreators = filters.second, - filterByLabelInfo = PipelineFilterByLabelInfo( - filterByLabels = filters.third, - labelToPipelineMap = filters.third.generateLabelToPipelineMap(projectId) - ) + + val pipelineIds = mutableSetOf() + val viewIdList = listOf( + PIPELINE_VIEW_FAVORITE_PIPELINES, + PIPELINE_VIEW_MY_PIPELINES, + PIPELINE_VIEW_ALL_PIPELINES, + PIPELINE_VIEW_MY_LIST_PIPELINES, + PIPELINE_VIEW_UNCLASSIFIED, + PIPELINE_VIEW_RECENT_USE + ) + val includeDelete = showDelete && (PIPELINE_VIEW_RECENT_USE == viewId || !viewIdList.contains(viewId)) + + if (!viewIdList.contains(viewId)) { // 已分组的视图 + pipelineIds.addAll(pipelineViewGroupService.listPipelineIdsByViewId(projectId, viewId)) + } else if (viewId == PIPELINE_VIEW_UNCLASSIFIED) { // 非分组的视图 + val allPipelineIds = pipelineInfoDao.listPipelineIdByProject(dslContext, projectId).toMutableSet() + pipelineIds.addAll( + allPipelineIds.subtract(pipelineViewGroupService.getClassifiedPipelineIds(projectId).toSet()) ) - pipelineFilterParamList.add(pipelineViewFilterParam) + // 避免过滤器为空的情况 + if (pipelineIds.isEmpty()) { + pipelineIds.add("##NONE##") + } + } else if (viewId == PIPELINE_VIEW_RECENT_USE) { // 最近访问 + pipelineIds.addAll(pipelineRecentUseService.listPipelineIds(userId, projectId)) + } + // 剔除掉filterByViewIds + if (filterByViewIds != null) { + val pipelineIdsByFilterViewIds = + pipelineViewGroupService.listPipelineIdsByViewIds(projectId, filterByViewIds.split(",")).toSet() + if (pipelineIds.isEmpty()) { + pipelineIds.addAll(pipelineIdsByFilterViewIds) + } else { + pipelineIds.retainAll(pipelineIdsByFilterViewIds) + } } + pipelineViewService.addUsingView(userId = userId, projectId = projectId, viewId = viewId) // 查询有权限查看的流水线总数 @@ -478,12 +520,14 @@ class PipelineListFacadeService @Autowired constructor( dslContext = dslContext, projectId = projectId, channelCode = channelCode, - pipelineIds = authPipelines, + pipelineIds = pipelineIds, favorPipelines = favorPipelines, authPipelines = authPipelines, viewId = viewId, pipelineFilterParamList = pipelineFilterParamList, - permissionFlag = true + permissionFlag = true, + includeDelete = includeDelete, + userId = userId ) // 查询无权限查看的流水线总数 @@ -492,15 +536,36 @@ class PipelineListFacadeService @Autowired constructor( dslContext = dslContext, projectId = projectId, channelCode = channelCode, + pipelineIds = pipelineIds, + viewId = viewId, favorPipelines = favorPipelines, authPipelines = authPipelines, - viewId = viewId, pipelineFilterParamList = pipelineFilterParamList, - permissionFlag = false + permissionFlag = false, + includeDelete = includeDelete, + userId = userId ) val pipelineList = mutableListOf() val totalSize = totalAvailablePipelineSize + totalInvalidPipelineSize - if ((null != page && null != pageSize) && !(page == 1 && pageSize == -1)) { + if (includeDelete) { + handlePipelineQueryList( + pipelineList = pipelineList, + projectId = projectId, + channelCode = channelCode, + sortType = sortType, + pipelineIds = pipelineIds, + favorPipelines = favorPipelines, + authPipelines = authPipelines, + viewId = viewId, + pipelineFilterParamList = pipelineFilterParamList, + permissionFlag = null, + page = page, + pageSize = pageSize, + includeDelete = true, + collation = collation, + userId = userId + ) + } else if ((null != page && null != pageSize) && !(page == 1 && pageSize == -1)) { // 判断可用的流水线是否已到最后一页 val totalAvailablePipelinePage = PageUtil.calTotalPage(pageSize, totalAvailablePipelineSize) if (page < totalAvailablePipelinePage) { @@ -510,13 +575,17 @@ class PipelineListFacadeService @Autowired constructor( projectId = projectId, channelCode = channelCode, sortType = sortType, + pipelineIds = pipelineIds, favorPipelines = favorPipelines, authPipelines = authPipelines, viewId = viewId, pipelineFilterParamList = pipelineFilterParamList, permissionFlag = true, page = page, - pageSize = pageSize + pageSize = pageSize, + includeDelete = includeDelete, + collation = collation, + userId = userId ) } else if (page == totalAvailablePipelinePage && totalAvailablePipelineSize > 0) { // 查询可用流水线最后一页不满页的数量 @@ -526,13 +595,17 @@ class PipelineListFacadeService @Autowired constructor( projectId = projectId, channelCode = channelCode, sortType = sortType, + pipelineIds = pipelineIds, favorPipelines = favorPipelines, authPipelines = authPipelines, viewId = viewId, pipelineFilterParamList = pipelineFilterParamList, permissionFlag = true, page = page, - pageSize = pageSize + pageSize = pageSize, + includeDelete = includeDelete, + collation = collation, + userId = userId ) // 可用流水线最后一页不满页的数量需用不可用的流水线填充 if (lastPageRemainNum > 0 && totalInvalidPipelineSize > 0) { @@ -541,13 +614,17 @@ class PipelineListFacadeService @Autowired constructor( projectId = projectId, channelCode = channelCode, sortType = sortType, + pipelineIds = pipelineIds, favorPipelines = favorPipelines, authPipelines = authPipelines, viewId = viewId, pipelineFilterParamList = pipelineFilterParamList, permissionFlag = false, page = 1, - pageSize = lastPageRemainNum.toInt() + pageSize = lastPageRemainNum.toInt(), + includeDelete = includeDelete, + collation = collation, + userId = userId ) } } else if (totalInvalidPipelineSize > 0) { @@ -559,6 +636,7 @@ class PipelineListFacadeService @Autowired constructor( projectId = projectId, channelCode = channelCode, sortType = sortType, + pipelineIds = pipelineIds, favorPipelines = favorPipelines, authPipelines = authPipelines, viewId = viewId, @@ -566,7 +644,10 @@ class PipelineListFacadeService @Autowired constructor( permissionFlag = false, page = page - totalAvailablePipelinePage, pageSize = pageSize, - pageOffsetNum = lastPageRemainNum.toInt() + pageOffsetNum = lastPageRemainNum.toInt(), + includeDelete = includeDelete, + collation = collation, + userId = userId ) } } else { @@ -576,13 +657,17 @@ class PipelineListFacadeService @Autowired constructor( projectId = projectId, channelCode = channelCode, sortType = sortType, + pipelineIds = pipelineIds, favorPipelines = favorPipelines, authPipelines = authPipelines, viewId = viewId, pipelineFilterParamList = pipelineFilterParamList, permissionFlag = true, page = page, - pageSize = pageSize + pageSize = pageSize, + includeDelete = includeDelete, + collation = collation, + userId = userId ) if (filterInvalid) { @@ -591,13 +676,17 @@ class PipelineListFacadeService @Autowired constructor( projectId = projectId, channelCode = channelCode, sortType = sortType, + pipelineIds = pipelineIds, favorPipelines = favorPipelines, authPipelines = authPipelines, viewId = viewId, pipelineFilterParamList = pipelineFilterParamList, permissionFlag = false, page = page, - pageSize = pageSize + pageSize = pageSize, + includeDelete = includeDelete, + collation = collation, + userId = userId ) } } @@ -615,6 +704,57 @@ class PipelineListFacadeService @Autowired constructor( } } + fun getCount(userId: String, projectId: String): PipelineCount { + val authPipelines = pipelinePermissionService.getResourceByPermission( + userId = userId, projectId = projectId, permission = AuthPermission.LIST + ) + val favorPipelines = pipelineGroupService.getFavorPipelines(userId = userId, projectId = projectId) + val recentUsePipelines = pipelineRecentUseService.listPipelineIds(userId, projectId) + val totalCount = pipelineBuildSummaryDao.listPipelineInfoBuildSummaryCount( + dslContext = dslContext, + projectId = projectId, + channelCode = ChannelCode.BS, + authPipelines = authPipelines, + favorPipelines = favorPipelines, + viewId = PIPELINE_VIEW_ALL_PIPELINES, + includeDelete = false, + userId = userId + ).toInt() + val myFavoriteCount = pipelineBuildSummaryDao.listPipelineInfoBuildSummaryCount( + dslContext = dslContext, + projectId = projectId, + channelCode = ChannelCode.BS, + authPipelines = authPipelines, + favorPipelines = favorPipelines, + viewId = PIPELINE_VIEW_FAVORITE_PIPELINES, + includeDelete = false, + userId = userId + ).toInt() + val myPipelineCount = pipelineBuildSummaryDao.listPipelineInfoBuildSummaryCount( + dslContext = dslContext, + projectId = projectId, + channelCode = ChannelCode.BS, + authPipelines = authPipelines, + favorPipelines = favorPipelines, + viewId = PIPELINE_VIEW_MY_PIPELINES, + includeDelete = false, + userId = userId + ).toInt() + val recentUseCount = pipelineBuildSummaryDao.listPipelineInfoBuildSummaryCount( + dslContext = dslContext, + projectId = projectId, + channelCode = ChannelCode.BS, + authPipelines = authPipelines, + favorPipelines = favorPipelines, + viewId = PIPELINE_VIEW_RECENT_USE, + includeDelete = false, + userId = userId, + pipelineIds = recentUsePipelines + ).toInt() + val recycleCount = pipelineInfoDao.countDeletePipeline(dslContext, projectId, deletedPipelineStoreDays.toLong()) + return PipelineCount(totalCount, myFavoriteCount, myPipelineCount, recycleCount, recentUseCount) + } + private fun handlePipelineQueryList( pipelineList: MutableList, projectId: String, @@ -628,7 +768,10 @@ class PipelineListFacadeService @Autowired constructor( permissionFlag: Boolean? = null, page: Int? = null, pageSize: Int? = null, - pageOffsetNum: Int? = 0 + pageOffsetNum: Int? = 0, + includeDelete: Boolean? = false, + collation: PipelineCollation = PipelineCollation.DEFAULT, + userId: String ) { val pipelineRecords = pipelineBuildSummaryDao.listPipelineInfoBuildSummary( dslContext = dslContext, @@ -643,7 +786,10 @@ class PipelineListFacadeService @Autowired constructor( permissionFlag = permissionFlag, page = page, pageSize = pageSize, - pageOffsetNum = pageOffsetNum + pageOffsetNum = pageOffsetNum, + includeDelete = includeDelete, + collation = collation, + userId = userId ) pipelineList.addAll( buildPipelines( @@ -675,17 +821,8 @@ class PipelineListFacadeService @Autowired constructor( pipelines: List, viewId: String ): List { - val view = pipelineViewService.getView(userId = userId, projectId = projectId, viewId = viewId) - val filters = pipelineViewService.getFilters(view) - - return filterViewPipelines( - projectId = projectId, - pipelines = pipelines, - logic = view.logic, - filterByPipelineNames = filters.first, - filterByPipelineCreators = filters.second, - filterByLabels = filters.third - ) + val pipelineIds = pipelineViewGroupService.listPipelineIdsByViewId(projectId, viewId) + return pipelines.filter { pipelineIds.contains(it.pipelineId) } } /** @@ -1107,12 +1244,39 @@ class PipelineListFacadeService @Autowired constructor( ).map { it.get(TPipelineSetting.T_PIPELINE_SETTING.PIPELINE_ID) to it }.toMap() // 获取summary信息 + val lastBuildMap = mutableMapOf() val pipelineBuildSummaryMap = pipelineBuildSummaryDao.listSummaryByPipelineIds( dslContext = dslContext, pipelineIds = pipelineIds, projectId = projectId + ).map { + if (null != it.latestBuildId) { + lastBuildMap[it.latestBuildId] = it.pipelineId + } + it.pipelineId to it + }.toMap() + + // 根据LastBuildId获取最新构建的信息 + val pipelineBuildMap = pipelineBuildDao.listBuildInfoByBuildIds( + dslContext = dslContext, + projectId = projectId, + buildIds = lastBuildMap.keys ).map { it.pipelineId to it }.toMap() + // 根据LastBuild获取运行中的构建任务个数 + val buildTaskCountList = pipelineBuildTaskDao.countGroupByBuildId( + dslContext = dslContext, + projectId = projectId, + buildIds = lastBuildMap.keys + ) + val buildTaskTotalCountMap = buildTaskCountList.groupBy { it.value1() } + .map { it -> lastBuildMap.getOrDefault(it.key, "") to it.value.sumOf { it.value3() } } + .toMap() + val buildTaskFinishCountMap = buildTaskCountList.filter { it.value2() == BuildStatus.SUCCEED.ordinal } + .groupBy { it.value1() } + .map { it -> lastBuildMap.getOrDefault(it.key, "") to it.value.sumOf { it.value3() } } + .toMap() + // 获取template信息 val tTemplate = TTemplatePipeline.T_TEMPLATE_PIPELINE val pipelineTemplateMap = templatePipelineDao.listSimpleByPipelines( @@ -1138,6 +1302,9 @@ class PipelineListFacadeService @Autowired constructor( emptyMap() } + // 获取view信息 + val pipelineViewNameMap = pipelineViewGroupService.getViewNameMap(projectId, pipelineIds) + // 完善数据 finalPipelines( pipelines = pipelines, @@ -1145,7 +1312,11 @@ class PipelineListFacadeService @Autowired constructor( pipelineTemplateMap = pipelineTemplateMap, pipelineGroupLabel = pipelineGroupLabel, pipelineBuildSummaryMap = pipelineBuildSummaryMap, - pipelineSettingMap = pipelineSettingMap + pipelineSettingMap = pipelineSettingMap, + pipelineViewNameMap = pipelineViewNameMap, + pipelineBuildMap = pipelineBuildMap, + buildTaskTotalCountMap = buildTaskTotalCountMap, + buildTaskFinishCountMap = buildTaskFinishCountMap ) return pipelines @@ -1157,7 +1328,11 @@ class PipelineListFacadeService @Autowired constructor( pipelineTemplateMap: Map, pipelineGroupLabel: Map>, pipelineBuildSummaryMap: Map, - pipelineSettingMap: Map> + pipelineSettingMap: Map>, + pipelineViewNameMap: Map>, + pipelineBuildMap: Map, + buildTaskTotalCountMap: Map, + buildTaskFinishCountMap: Map ) { pipelines.forEach { val pipelineId = it.pipelineId @@ -1184,7 +1359,40 @@ class PipelineListFacadeService @Autowired constructor( it.latestBuildId = pipelineBuildSummaryRecord.latestBuildId it.latestBuildUserId = pipelineBuildSummaryRecord.latestStartUser ?: "" it.latestBuildNumAlias = pipelineBuildSummaryRecord.buildNumAlias + it.viewNames = pipelineViewNameMap[it.pipelineId] } + pipelineBuildMap[pipelineId]?.let { lastBuild -> + it.lastBuildMsg = lastBuild.buildMsg + it.trigger = lastBuild.trigger + val webhookInfo = lastBuild.webhookInfo?.let { self -> + JsonUtil.to(self, object : TypeReference() {}) + } + if (webhookInfo != null) { + it.webhookAliasName = webhookInfo.webhookAliasName ?: getProjectName(webhookInfo.webhookRepoUrl) + it.webhookRepoUrl = webhookInfo.webhookRepoUrl + it.webhookType = it.webhookType + val eventType = try { + webhookInfo.webhookEventType?.let { e -> CodeEventType.valueOf(e) } + } catch (e: Exception) { + null + } + it.webhookMessage = when (eventType) { + CodeEventType.PUSH -> webhookInfo.webhookCommitId?.let { e -> + val endIndex = e.length.coerceAtMost(7) + "Commit [${e.substring(0, endIndex)}] pushed" + } + + CodeEventType.MERGE_REQUEST -> webhookInfo.mrIid?.let { e -> "Merge requests [!$e] open" } + CodeEventType.TAG_PUSH -> webhookInfo.tagName?.let { e -> "Tag [$e] pushed" } + CodeEventType.ISSUES -> webhookInfo.issueIid?.let { e -> "Issue [$e] opened" } + CodeEventType.NOTE -> webhookInfo.noteId?.let { e -> "Note [$e] submitted" } + CodeEventType.REVIEW -> webhookInfo.reviewId?.let { e -> "Review [$e] created" } + else -> null + } + } + } + it.lastBuildFinishCount = buildTaskFinishCountMap.getOrDefault(pipelineId, 0) + it.lastBuildTotalCount = buildTaskTotalCountMap.getOrDefault(pipelineId, 0) val pipelineSettingRecord = pipelineSettingMap[pipelineId] if (pipelineSettingRecord != null) { val tSetting = TPipelineSetting.T_PIPELINE_SETTING @@ -1195,6 +1403,17 @@ class PipelineListFacadeService @Autowired constructor( } } + private fun getProjectName(webhookRepoUrl: String?): String { + if (null == webhookRepoUrl) { + return "" + } + return try { + GitUtils.getProjectName(webhookRepoUrl) + } catch (e: Exception) { + webhookRepoUrl + } + } + private fun initPipelines( pipelineInfoRecords: Result, excludePipelineId: String?, @@ -1227,7 +1446,8 @@ class PipelineListFacadeService @Autowired constructor( hasPermission = authPipelines.contains(pipelineId), hasCollect = favorPipelines.contains(pipelineId), updater = it.lastModifyUser, - creator = it.creator + creator = it.creator, + delete = it.delete ) ) } @@ -1292,31 +1512,32 @@ class PipelineListFacadeService @Autowired constructor( projectId: String, page: Int?, pageSize: Int?, - sortType: PipelineSortType?, - channelCode: ChannelCode + sortType: PipelineSortType, + channelCode: ChannelCode, + collation: PipelineCollation ): PipelineViewPipelinePage { val pageNotNull = page ?: 0 val pageSizeNotNull = pageSize ?: -1 - var slqLimit: SQLLimit? = null - if (pageSizeNotNull != -1) slqLimit = PageUtil.convertPageSizeToSQLLimit(pageNotNull, pageSizeNotNull) + val slqLimit: SQLLimit = PageUtil.convertPageSizeToSQLLimit(pageNotNull, pageSizeNotNull) - val offset = slqLimit?.offset ?: 0 - val limit = slqLimit?.limit ?: -1 - // 数据量不多，直接全拉 - val pipelines = - pipelineRepositoryService.listDeletePipelineIdByProject(projectId, deletedPipelineStoreDays.toLong()) - val list: List = when { - offset >= pipelines.size -> emptyList() - limit < 0 -> pipelines.subList(offset, pipelines.size) - else -> { - val toIndex = if (pipelines.size <= (offset + limit)) pipelines.size else offset + limit - pipelines.subList(offset, toIndex) - } - } + // 获取列表和数目 + val list = pipelineRepositoryService.listDeletePipelineIdByProject( + projectId = projectId, + days = deletedPipelineStoreDays.toLong(), + offset = slqLimit.offset, + limit = slqLimit.limit, + sortType = sortType, + collation = collation + ) + val count = pipelineInfoDao.countDeletePipeline(dslContext, projectId, deletedPipelineStoreDays.toLong()) + // 加上流水线组 + val pipelineViewNameMap = + pipelineViewGroupService.getViewNameMap(projectId, list.map { it.pipelineId }.toMutableSet()) + list.forEach { it.viewNames = pipelineViewNameMap[it.pipelineId] } return PipelineViewPipelinePage( page = pageNotNull, pageSize = pageSizeNotNull, - count = pipelines.size + 0L, + count = count.toLong(), records = list ) } @@ -1424,7 +1645,8 @@ class PipelineListFacadeService @Autowired constructor( val pipelineInfos = mutableListOf() pipelineRecords?.map { pipelineInfos.add( - PipelineIdAndName(it.pipelineId, it.pipelineName)) + PipelineIdAndName(it.pipelineId, it.pipelineName) + ) } val count = pipelineInfoDao.countByProjectIds( dslContext = dslContext, @@ -1689,16 +1911,19 @@ class PipelineListFacadeService @Autowired constructor( logger.info("User($userId) favorite pipeline ids($favorPipelines)") allFilterPipelines.filter { favorPipelines.contains(it.pipelineId) } } + PIPELINE_VIEW_MY_PIPELINES -> { logger.info("User($userId) my pipelines") allFilterPipelines.filter { authPipelines.contains(it.pipelineId) } } + PIPELINE_VIEW_ALL_PIPELINES -> { logger.info("User($userId) all pipelines") allFilterPipelines } + else -> { logger.info("User($userId) filter view($viewId)") filterViewPipelines(userId, projectId, allFilterPipelines, viewId) diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineRecentUseService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineRecentUseService.kt new file mode 100644 index 00000000000..2d0e5189372 --- /dev/null +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineRecentUseService.kt @@ -0,0 +1,32 @@ +package com.tencent.devops.process.service + +import com.tencent.devops.process.dao.PipelineRecentUseDao +import org.jooq.DSLContext +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +/** + * 最近使用流水线服务 + */ +@Service +class PipelineRecentUseService @Autowired constructor( + private val pipelineRecentUseDao: PipelineRecentUseDao, + private val dslContext: DSLContext +) { + + fun listPipelineIds(userId: String, projectId: String, noEmpty: Boolean = true): List { + val pipelineIds = pipelineRecentUseDao.listRecentPipelineIds(dslContext, projectId, userId, RECENT_USE_LIST_MAX) + if (noEmpty && pipelineIds.isEmpty()) { + return listOf("##NONE##") + } + return pipelineIds + } + + fun record(userId: String, projectId: String, pipelineId: String) { + pipelineRecentUseDao.add(dslContext, projectId, userId, pipelineId) + } + + companion object { + private const val RECENT_USE_LIST_MAX = 30 + } +} diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt index 59d4b9a8225..b573db47672 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt @@ -31,6 +31,7 @@ import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.exception.ParamBlankException import com.tencent.devops.common.api.model.SQLPage import com.tencent.devops.common.api.pojo.BuildHistoryPage +import com.tencent.devops.common.api.pojo.ErrorType import com.tencent.devops.common.api.pojo.IdValue import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.api.pojo.SimpleResult @@ -414,13 +415,17 @@ class PipelineBuildFacadeService( model.stages.forEach { s -> // stage 级重试 if (s.id == taskId) { - pipelineStageService.getStage(projectId, buildId, stageId = s.id) ?: run { + val stage = pipelineStageService.getStage(projectId, buildId, stageId = s.id) ?: run { throw ErrorCodeException( errorCode = ProcessMessageCode.ERROR_BUILD_EXPIRED_CANT_RETRY, defaultMessage = "构建数据已过期，请使用rebuild进行重试(Please use rebuild)" ) } - + // 只有失败或取消情况下提供重试得可能 + if (!stage.status.isFailure() && !stage.status.isCancel()) throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_RETRY_STAGE_NOT_FAILED, + defaultMessage = "Stage($taskId)未处于失败或取消状态，无法重试" + ) paramMap[PIPELINE_RETRY_START_TASK_ID] = BuildParameters( key = PIPELINE_RETRY_START_TASK_ID, value = s.id!! ) @@ -498,7 +503,7 @@ class PipelineBuildFacadeService( logger.info( "ENGINE|$buildId|RETRY_PIPELINE_ORIGIN|taskId=$taskId|$pipelineId|" + - "retryCount=$retryCount|fc=$failedContainer|skip=$skipFailedTask" + "retryCount=$retryCount|fc=$failedContainer|skip=$skipFailedTask" ) paramMap[PIPELINE_RETRY_COUNT] = BuildParameters(PIPELINE_RETRY_COUNT, retryCount) @@ -1349,6 +1354,9 @@ class PipelineBuildFacadeService( defaultMessage = "构建任务${buildId}不存在", params = arrayOf(buildId) ) + val currentQueuePosition = if (BuildStatus.valueOf(buildHistory.status).isReadyToRun()) { + getCurrentQueuePosition(buildHistory, projectId, pipelineId) + } else 0 val variables = buildVariableService.getAllVariable(projectId, pipelineId, buildId) return BuildHistoryWithVars( @@ -1365,6 +1373,7 @@ class PipelineBuildFacadeService( isMobileStart = buildHistory.isMobileStart, material = buildHistory.material, queueTime = buildHistory.queueTime, + currentQueuePosition = currentQueuePosition, artifactList = buildHistory.artifactList, remark = buildHistory.remark, totalTime = buildHistory.totalTime, @@ -1382,6 +1391,28 @@ class PipelineBuildFacadeService( ) } + /** + * 拿排队位置，分两种排队。GROUP_LOCK 排队只算当前并发组、 LOCK排队只算当前流水线。 + */ + private fun getCurrentQueuePosition( + buildHistory: BuildHistory, + projectId: String, + pipelineId: String + ) = if (!buildHistory.concurrencyGroup.isNullOrBlank()) { + pipelineRuntimeService.getBuildInfoListByConcurrencyGroup( + projectId = projectId, + concurrencyGroup = buildHistory.concurrencyGroup!!, + status = listOf(BuildStatus.QUEUE, BuildStatus.QUEUE_CACHE) + ).indexOfFirst { it.second == buildHistory.id } + 1 + } else { + pipelineRuntimeService.getPipelineBuildHistoryCount( + projectId = projectId, + pipelineId = pipelineId, + status = listOf(BuildStatus.QUEUE, BuildStatus.QUEUE_CACHE), + startTimeEndTime = buildHistory.startTime + ) + } + fun getBuildVars( userId: String, projectId: String, @@ -1471,6 +1502,18 @@ class PipelineBuildFacadeService( return buildHistories } + fun getBuilds( + userId: String, + projectId: String, + pipelineId: String?, + buildStatus: Set?, + checkPermission: Boolean + ): List { + return pipelineRuntimeService.getBuilds( + projectId, pipelineId, buildStatus + ) + } + fun getHistoryBuild( userId: String?, projectId: String, @@ -1981,6 +2024,12 @@ class PipelineBuildFacadeService( msg = "$msg| ${MessageCodeUtil.getCodeLanMessage(ProcessMessageCode.BUILD_WORKER_DEAD_ERROR)}" } + // 添加错误码日志 + val realErrorType = ErrorType.getErrorType(simpleResult.error?.errorType) + simpleResult.error?.errorType.let { msg = "$msg \nerrorType: ${realErrorType?.typeName}" } + simpleResult.error?.errorCode.let { msg = "$msg \nerrorCode: ${simpleResult.error?.errorCode}" } + simpleResult.error?.errorMessage.let { msg = "$msg \nerrorMsg: ${simpleResult.error?.errorMessage}" } + val buildInfo = pipelineRuntimeService.getBuildInfo(projectCode, buildId) if (buildInfo == null || buildInfo.status.isFinish()) { logger.warn("[$buildId]|workerBuildFinish|The build status is ${buildInfo?.status}") @@ -2013,7 +2062,9 @@ class PipelineBuildFacadeService( containerHashId = container.containerHashId, containerType = container.containerType, actionType = ActionType.TERMINATE, - reason = msg + reason = msg, + errorCode = simpleResult.error?.errorCode ?: 0, + errorTypeName = realErrorType?.typeName ) ) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingFacadeService.kt index fb0058bbdc5..4a1acf86224 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingFacadeService.kt @@ -29,6 +29,7 @@ package com.tencent.devops.process.service.pipeline import com.tencent.devops.common.api.constant.KEY_DEFAULT import com.tencent.devops.common.api.exception.PermissionForbiddenException +import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.common.client.Client import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher @@ -43,7 +44,6 @@ import com.tencent.devops.process.pojo.config.PipelineCommonSettingConfig import com.tencent.devops.process.pojo.config.StageCommonSettingConfig import com.tencent.devops.process.pojo.config.TaskCommonSettingConfig import com.tencent.devops.process.pojo.setting.JobCommonSetting -import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings import com.tencent.devops.process.pojo.setting.PipelineCommonSetting import com.tencent.devops.process.pojo.setting.PipelineRunLockType import com.tencent.devops.process.pojo.setting.PipelineSetting @@ -54,6 +54,7 @@ import com.tencent.devops.process.pojo.setting.TaskComponentCommonSetting import com.tencent.devops.process.pojo.setting.UpdatePipelineModelRequest import com.tencent.devops.process.service.PipelineSettingVersionService import com.tencent.devops.process.service.label.PipelineGroupService +import com.tencent.devops.process.service.view.PipelineViewGroupService import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -64,6 +65,7 @@ class PipelineSettingFacadeService @Autowired constructor( private val pipelineRepositoryService: PipelineRepositoryService, private val pipelineGroupService: PipelineGroupService, private val pipelineSettingVersionService: PipelineSettingVersionService, + private val pipelineViewGroupService: PipelineViewGroupService, private val pipelineCommonSettingConfig: PipelineCommonSettingConfig, private val stageCommonSettingConfig: StageCommonSettingConfig, private val jobCommonSettingConfig: JobCommonSettingConfig, @@ -78,7 +80,8 @@ class PipelineSettingFacadeService @Autowired constructor( checkPermission: Boolean = true, version: Int = 0, updateLastModifyUser: Boolean? = true, - dispatchPipelineUpdateEvent: Boolean = true + dispatchPipelineUpdateEvent: Boolean = true, + updateLabels: Boolean = true ): String { if (checkPermission) { checkEditPermission( @@ -104,12 +107,18 @@ class PipelineSettingFacadeService @Autowired constructor( ) } - pipelineGroupService.updatePipelineLabel( - userId = userId, - projectId = setting.projectId, - pipelineId = setting.pipelineId, - labelIds = setting.labels - ) + if (updateLabels) { + pipelineGroupService.updatePipelineLabel( + userId = userId, + projectId = setting.projectId, + pipelineId = setting.pipelineId, + labelIds = setting.labels + ) + } + + // 刷新流水线组 + pipelineViewGroupService.updateGroupAfterPipelineUpdate(setting.projectId, setting.pipelineId, userId) + if (dispatchPipelineUpdateEvent) { pipelineEventDispatcher.dispatch( PipelineUpdateEvent( diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt index af843e580e9..d799df3d892 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt @@ -182,6 +182,9 @@ class TemplateFacadeService @Autowired constructor( @Value("\${template.instanceListUrl}") private val instanceListUrl: String = "" + @Value("\${template.maxErrorReasonLength:200}") + private val maxErrorReasonLength: Int = 200 + fun createTemplate(projectId: String, userId: String, template: Model): String { logger.info("Start to create the template ${template.name} by user $userId") checkPermission(projectId, userId) @@ -1418,7 +1421,7 @@ class TemplateFacadeService @Autowired constructor( val validateRet = client.get(ServiceTemplateResource::class) .validateUserTemplateComponentVisibleDept( userId = userId, - templateCode = templateId, + templateCode = srcTemplateId, projectCode = projectId ) if (validateRet.isNotOk()) { @@ -1521,8 +1524,8 @@ class TemplateFacadeService @Autowired constructor( } catch (t: Throwable) { logger.warn("FailUpdateTemplate|${templateInstanceUpdate.pipelineName}|$projectId|$userId", t) val message = - if (!t.message.isNullOrBlank() && t.message!!.length > 36) - t.message!!.substring(0, 36) + "......" else t.message + if (!t.message.isNullOrBlank() && t.message!!.length > maxErrorReasonLength) + t.message!!.substring(0, maxErrorReasonLength) + "......" else t.message failurePipelines.add("【${templateInstanceUpdate.pipelineName}】reason：$message") } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/view/PipelineViewGroupService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/view/PipelineViewGroupService.kt new file mode 100644 index 00000000000..33984e7b797 --- /dev/null +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/view/PipelineViewGroupService.kt @@ -0,0 +1,761 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.service.view + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.github.benmanes.caffeine.cache.Caffeine +import com.tencent.devops.auth.api.service.ServiceProjectAuthResource +import com.tencent.devops.common.api.exception.ErrorCodeException +import com.tencent.devops.common.api.util.HashUtil +import com.tencent.devops.common.api.util.Watcher +import com.tencent.devops.common.api.util.timestamp +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.client.ClientTokenService +import com.tencent.devops.common.pipeline.enums.ChannelCode +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.common.service.utils.LogUtils +import com.tencent.devops.model.process.tables.records.TPipelineInfoRecord +import com.tencent.devops.model.process.tables.records.TPipelineViewRecord +import com.tencent.devops.process.constant.PipelineViewType +import com.tencent.devops.process.constant.ProcessMessageCode +import com.tencent.devops.process.dao.label.PipelineViewDao +import com.tencent.devops.process.dao.label.PipelineViewGroupDao +import com.tencent.devops.process.dao.label.PipelineViewTopDao +import com.tencent.devops.process.engine.dao.PipelineInfoDao +import com.tencent.devops.process.pojo.classify.PipelineNewView +import com.tencent.devops.process.pojo.classify.PipelineNewViewSummary +import com.tencent.devops.process.pojo.classify.PipelineViewBulkAdd +import com.tencent.devops.process.pojo.classify.PipelineViewBulkRemove +import com.tencent.devops.process.pojo.classify.PipelineViewDict +import com.tencent.devops.process.pojo.classify.PipelineViewFilter +import com.tencent.devops.process.pojo.classify.PipelineViewForm +import com.tencent.devops.process.pojo.classify.PipelineViewPipelineCount +import com.tencent.devops.process.pojo.classify.PipelineViewPreview +import com.tencent.devops.process.pojo.classify.enums.Logic +import com.tencent.devops.process.service.view.lock.PipelineViewGroupLock +import com.tencent.devops.process.utils.PIPELINE_VIEW_UNCLASSIFIED +import org.apache.commons.lang3.StringUtils +import org.jooq.DSLContext +import org.jooq.impl.DSL +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import java.time.LocalDateTime +import java.util.concurrent.TimeUnit + +@Service +@SuppressWarnings("LoopWithTooManyJumpStatements", "LongParameterList", "TooManyFunctions", "ReturnCount") +class PipelineViewGroupService @Autowired constructor( + private val pipelineViewService: PipelineViewService, + private val pipelineViewDao: PipelineViewDao, + private val pipelineViewGroupDao: PipelineViewGroupDao, + private val pipelineViewTopDao: PipelineViewTopDao, + private val pipelineInfoDao: PipelineInfoDao, + private val dslContext: DSLContext, + private val redisOperation: RedisOperation, + private val objectMapper: ObjectMapper, + private val client: Client, + private val clientTokenService: ClientTokenService +) { + private val allPipelineInfoCache = Caffeine.newBuilder() + .maximumSize(10) + .expireAfterWrite(10, TimeUnit.SECONDS) + .build>() + + fun getViewNameMap( + projectId: String, + pipelineIds: MutableSet + ): Map/*viewNames*/> { + val pipelineViewGroups = pipelineViewGroupDao.listByPipelineIds(dslContext, projectId, pipelineIds) + if (pipelineViewGroups.isEmpty()) { + return emptyMap() + } + val viewIds = pipelineViewGroups.map { it.viewId }.toSet() + val views = pipelineViewDao.list(dslContext, projectId, viewIds) + if (views.isEmpty()) { + return emptyMap() + } + val viewId2Name = views.filter { it.isProject }.associate { it.id to it.name } + val result = mutableMapOf>() + for (p in pipelineViewGroups) { + if (!viewId2Name.containsKey(p.viewId)) { + continue + } + if (!result.containsKey(p.pipelineId)) { + result[p.pipelineId] = mutableListOf() + } + result[p.pipelineId]!!.add(viewId2Name[p.viewId]!!) + } + + return result + } + + fun addViewGroup(projectId: String, userId: String, pipelineView: PipelineViewForm): String { + checkPermission(userId, projectId, pipelineView.projected) + var viewId = 0L + dslContext.transaction { t -> + val context = DSL.using(t) + viewId = pipelineViewService.addView(userId, projectId, pipelineView, context) + initViewGroup( + context = context, + pipelineView = pipelineView, + projectId = projectId, + viewId = viewId, + userId = userId + ) + } + return HashUtil.encodeLongId(viewId) + } + + fun updateViewGroup( + projectId: String, + userId: String, + viewIdEncode: String, + pipelineView: PipelineViewForm + ): Boolean { + // 获取老视图 + val viewId = HashUtil.decodeIdToLong(viewIdEncode) + val oldView = pipelineViewDao.get(dslContext, projectId, viewId) ?: throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_VIEW_NOT_FOUND, + params = arrayOf(viewIdEncode) + ) + // 校验 + checkPermission(userId, projectId, pipelineView.projected, oldView.createUser) + if (pipelineView.projected != oldView.isProject) { + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_VIEW_GROUP_IS_PROJECT_NO_SAME, + defaultMessage = "view scope can`t change , user:$userId , view:$viewIdEncode , project:$projectId" + ) + } + if (pipelineView.viewType == PipelineViewType.UNCLASSIFIED) { + pipelineView.viewType = oldView.viewType + } + // 更新视图 + var result = false + dslContext.transaction { t -> + val context = DSL.using(t) + result = pipelineViewService.updateView(userId, projectId, viewId, pipelineView, context) + if (result) { + if (pipelineView.pipelineIds != null) { + pipelineViewGroupDao.remove(context, projectId, viewId) + } + redisOperation.delete(firstInitMark(projectId, viewId)) + initViewGroup( + context = context, + pipelineView = pipelineView, + projectId = projectId, + viewId = viewId, + userId = userId + ) + } + } + return result + } + + fun getView(userId: String, projectId: String, viewId: String): PipelineNewView { + val viewRecord = pipelineViewDao.get( + dslContext = dslContext, + projectId = projectId, + viewId = HashUtil.decodeIdToLong(viewId) + ) ?: throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_VIEW_NOT_FOUND, + params = arrayOf(viewId) + ) + + val filters = pipelineViewService.getFilters( + filterByName = viewRecord.filterByPipeineName, + filterByCreator = viewRecord.filterByCreator, + filters = viewRecord.filters + ) + + val pipelineIds = pipelineViewGroupDao.listByViewId(dslContext, projectId, viewRecord.id).map { it.pipelineId } + + return PipelineNewView( + id = viewId, + projectId = viewRecord.projectId, + name = viewRecord.name, + projected = viewRecord.isProject, + createTime = viewRecord.createTime.timestamp(), + updateTime = viewRecord.updateTime.timestamp(), + creator = viewRecord.createUser, + logic = Logic.of(viewRecord.logic), + filters = filters, + viewType = viewRecord.viewType, + pipelineIds = pipelineIds + ) + } + + fun deleteViewGroup( + projectId: String, + userId: String, + viewIdEncode: String + ): Boolean { + val viewId = HashUtil.decodeIdToLong(viewIdEncode) + val oldView = pipelineViewDao.get(dslContext, projectId, viewId) ?: throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_VIEW_NOT_FOUND, + params = arrayOf(viewIdEncode) + ) + checkPermission(userId, projectId, oldView.isProject, oldView.createUser) + var result = false + dslContext.transaction { t -> + val context = DSL.using(t) + result = pipelineViewService.deleteView(userId, projectId, viewId) + if (result) { + pipelineViewGroupDao.remove(context, projectId, viewId) + } + } + return result + } + + fun getClassifiedPipelineIds(projectId: String): List { + val projectViews = pipelineViewDao.list(dslContext = dslContext, projectId = projectId, isProject = true) + if (projectViews.isEmpty()) { + return emptyList() + } + return pipelineViewGroupDao.distinctPipelineIds(dslContext, projectId, projectViews.map { it.id }) + } + + fun listPipelineIdsByViewIds(projectId: String, viewIdsEncode: List): List { + val viewIds = viewIdsEncode.map { HashUtil.decodeIdToLong(it) } + val pipelineIds = mutableListOf() + val viewGroups = pipelineViewGroupDao.listByViewIds(dslContext, projectId, viewIds) + if (viewGroups.isEmpty()) { + pipelineIds.addAll(emptyList()) + } else { + pipelineIds.addAll(viewGroups.map { it.pipelineId }.toList()) + } + if (pipelineIds.isEmpty()) { + pipelineIds.add("##NONE##") // 特殊标志,避免有些判空逻辑导致过滤器没有执行 + } + return pipelineIds + } + + fun listPipelineIdsByViewId(projectId: String, viewIdEncode: String): List { + return listPipelineIdsByViewIds(projectId, listOf(viewIdEncode)) + } + + fun updateGroupAfterPipelineCreate(projectId: String, pipelineId: String, userId: String) { + PipelineViewGroupLock(redisOperation, projectId).lockAround { + logger.info("updateGroupAfterPipelineCreate, projectId:$projectId, pipelineId:$pipelineId , userId:$userId") + val pipelineInfo = pipelineInfoDao.getPipelineId(dslContext, projectId, pipelineId)!! + val viewGroupCount = + pipelineViewGroupDao.countByPipelineId(dslContext, pipelineInfo.projectId, pipelineInfo.pipelineId) + if (viewGroupCount == 0) { + val dynamicProjectViews = + pipelineViewDao.list(dslContext, pipelineInfo.projectId, PipelineViewType.DYNAMIC) + val matchViewIds = dynamicProjectViews.asSequence() + .filter { pipelineViewService.matchView(it, pipelineInfo) } + .map { it.id } + .toSet() + matchViewIds.forEach { + pipelineViewGroupDao.create( + dslContext = dslContext, + projectId = projectId, + pipelineId = pipelineId, + viewId = it, + userId = userId + ) + } + } + } + } + + fun updateGroupAfterPipelineUpdate(projectId: String, pipelineId: String, userId: String) { + PipelineViewGroupLock(redisOperation, projectId).lockAround { + logger.info("updateGroupAfterPipelineUpdate, projectId:$projectId, pipelineId:$pipelineId , userId:$userId") + val pipelineInfo = pipelineInfoDao.getPipelineId(dslContext, projectId, pipelineId)!! + // 所有的动态项目组 + val dynamicProjectViews = pipelineViewDao.list(dslContext, pipelineInfo.projectId, PipelineViewType.DYNAMIC) + val dynamicProjectViewIds = dynamicProjectViews.asSequence() + .map { it.id } + .toSet() + // 命中的动态项目组 + val matchViewIds = dynamicProjectViews.asSequence() + .filter { pipelineViewService.matchView(it, pipelineInfo) } + .map { it.id } + .toSet() + // 已有的动态项目组 + val baseViewGroups = + pipelineViewGroupDao.listByPipelineId(dslContext, pipelineInfo.projectId, pipelineInfo.pipelineId) + .filter { dynamicProjectViewIds.contains(it.viewId) } + .toSet() + val baseViewIds = baseViewGroups.map { it.viewId }.toSet() + // 新增新命中的项目组 + matchViewIds.filterNot { baseViewIds.contains(it) }.forEach { + pipelineViewGroupDao.create( + dslContext = dslContext, + projectId = projectId, + pipelineId = pipelineId, + viewId = it, + userId = userId + ) + } + // 删除未命中的老项目组 + baseViewGroups.filterNot { matchViewIds.contains(it.viewId) }.forEach { + pipelineViewGroupDao.remove(dslContext, it.projectId, it.viewId, it.pipelineId) + } + } + } + + private fun initViewGroup( + context: DSLContext, + pipelineView: PipelineViewForm, + projectId: String, + viewId: Long, + userId: String + ) { + val watcher = Watcher("initViewGroup|$projectId|$viewId|$userId") + if (pipelineView.viewType == PipelineViewType.DYNAMIC) { + watcher.start("initDynamicViewGroup") + initDynamicViewGroup(pipelineViewDao.get(context, projectId, viewId)!!, userId, context) + watcher.stop() + } else { + watcher.start("initStaticViewGroup") + pipelineView.pipelineIds?.forEach { + pipelineViewGroupDao.create( + dslContext = context, + projectId = projectId, + pipelineId = it, + viewId = viewId, + userId = userId + ) + } + watcher.stop() + } + LogUtils.printCostTimeWE(watcher) + } + + private fun initDynamicViewGroup( + view: TPipelineViewRecord, + userId: String, + context: DSLContext? = null + ): List { + val projectId = view.projectId + val firstInit = redisOperation.setIfAbsent(firstInitMark(projectId, view.id), "1", 30 * 24 * 3600, true) + if (!firstInit) { + return emptyList() + } + return PipelineViewGroupLock(redisOperation, projectId).lockAround { + val pipelineIds = allPipelineInfos(projectId, false) + .filter { pipelineViewService.matchView(view, it) } + .map { it.pipelineId } + pipelineIds.forEach { + pipelineViewGroupDao.create( + dslContext = context ?: dslContext, + projectId = projectId, + pipelineId = it, + viewId = view.id, + userId = userId + ) + } + return@lockAround pipelineIds + } + } + + private fun firstInitMark( + projectId: String?, + viewId: Long + ) = "initDynamicViewGroup:$projectId:$viewId" + + private fun checkPermission(userId: String, projectId: String, isProject: Boolean, creator: String? = null) { + if (isProject) { + if (!hasPermission(userId, projectId)) { + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_VIEW_GROUP_NO_PERMISSION, + defaultMessage = "user:$userId has no permission to edit view group, project:$projectId" + ) + } + } else { + if (creator != null && userId != creator) { + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_DEL_PIPELINE_VIEW_NO_PERM, + defaultMessage = "user:$userId has no permission to edit view group, project:$projectId" + ) + } + } + } + + fun preview( + userId: String, + projectId: String, + pipelineView: PipelineViewForm + ): PipelineViewPreview { + // 获取所有流水线信息 + val allPipelineInfoMap = allPipelineInfos(projectId, true).associateBy { it.pipelineId } + if (allPipelineInfoMap.isEmpty()) { + return PipelineViewPreview.EMPTY + } + + // 获取老流水线组的流水线 + val viewId = pipelineView.id + val oldPipelineIds = if (null == viewId) { + emptyList() + } else { + pipelineViewGroupDao + .listByViewId(dslContext, projectId, HashUtil.decodeIdToLong(viewId)) + .map { it.pipelineId } + .filter { allPipelineInfoMap.containsKey(it) } + } + + // 获取新流水线组的流水线 + val newPipelineIds = if (pipelineView.viewType == PipelineViewType.DYNAMIC) { + val previewCondition = TPipelineViewRecord() + previewCondition.logic = pipelineView.logic.name + previewCondition.filterByPipeineName = StringUtils.EMPTY + previewCondition.filterByCreator = StringUtils.EMPTY + previewCondition.filters = objectMapper + .writerFor(object : TypeReference>() {}) + .writeValueAsString(pipelineView.filters) + allPipelineInfoMap.values + .filter { it.delete == false && pipelineViewService.matchView(previewCondition, it) } + .map { it.pipelineId } + } else { + pipelineView.pipelineIds?.filter { allPipelineInfoMap.containsKey(it) } ?: emptyList() + } + + // 新增流水线 = 新流水线 - 老流水线 + val addedPipelineInfos = newPipelineIds.asSequence() + .filterNot { oldPipelineIds.contains(it) } + .map { allPipelineInfoMap[it]!! } + .map { pipelineRecord2Info(it) } + .toList() + + // 移除流水线 = 老流水线 - 新流水线 + val removedPipelineInfos = oldPipelineIds.asSequence() + .filterNot { newPipelineIds.contains(it) } + .map { allPipelineInfoMap[it]!! } + .map { pipelineRecord2Info(it) } + .toList() + + // 保留流水线 = 老流水线 & 新流水线 + val reservePipelineInfos = newPipelineIds.asSequence() + .filter { oldPipelineIds.contains(it) } + .map { allPipelineInfoMap[it]!! } + .map { pipelineRecord2Info(it) } + .toList() + + return PipelineViewPreview(addedPipelineInfos, removedPipelineInfos, reservePipelineInfos) + } + + @SuppressWarnings("LongMethod", "ComplexMethod") + fun dict(userId: String, projectId: String): PipelineViewDict { + // 流水线信息 + val pipelineInfoMap = allPipelineInfos(projectId, true).associateBy { it.pipelineId } + if (pipelineInfoMap.isEmpty()) { + return PipelineViewDict.EMPTY + } + // 流水线组信息 + val viewInfoMap = pipelineViewDao.list(dslContext, projectId).associateBy { it.id } + // 流水线组映射关系 + val viewGroups = pipelineViewGroupDao.listByProjectId(dslContext, projectId) + val viewGroupMap = viewInfoMap.map { it.key to mutableListOf() }.toMap().toMutableMap() + val classifiedPipelineIds = mutableSetOf() + + viewGroups.forEach { + val viewId = it.viewId + if (!viewInfoMap.containsKey(viewId)) { + return@forEach + } + viewGroupMap[viewId]!!.add(it.pipelineId) + if (viewInfoMap[it.viewId]?.isProject == true) { + classifiedPipelineIds.add(it.pipelineId) + } + } + val personalViewList = mutableListOf() + val projectViewList = mutableListOf() + // 未分组数据加入 + projectViewList.add( + PipelineViewDict.ViewInfo( + viewId = PIPELINE_VIEW_UNCLASSIFIED, + viewName = "未分组", + pipelineList = pipelineInfoMap.values + .filterNot { classifiedPipelineIds.contains(it.pipelineId) } + .map { + PipelineViewDict.ViewInfo.PipelineInfo( + pipelineId = it.pipelineId, + pipelineName = it.pipelineName, + viewId = PIPELINE_VIEW_UNCLASSIFIED, + delete = it.delete + ) + } + ) + ) + // 拼装返回结果 + for (view in viewInfoMap.values) { + if (!view.isProject && view.createUser != userId) { + continue + } + if (!viewGroupMap.containsKey(view.id)) { + continue + } + val viewId = HashUtil.encodeLongId(view.id) + val pipelineList = viewGroupMap[view.id]!!.filter { pipelineInfoMap.containsKey(it) }.map { + val pipelineInfo = pipelineInfoMap[it]!! + PipelineViewDict.ViewInfo.PipelineInfo( + pipelineId = pipelineInfo.pipelineId, + pipelineName = pipelineInfo.pipelineName, + viewId = viewId, + delete = pipelineInfo.delete + ) + } + val viewList = if (view.isProject) projectViewList else personalViewList + viewList.add( + PipelineViewDict.ViewInfo( + viewId = viewId, + viewName = view.name, + pipelineList = pipelineList + ) + ) + } + return PipelineViewDict(personalViewList, projectViewList) + } + + private fun allPipelineInfos(projectId: String, includeDelete: Boolean): List { + return allPipelineInfoCache.get("$projectId-$includeDelete") { + val pipelineInfos = mutableListOf() + val step = 200 + var offset = 0 + var hasNext = true + while (hasNext) { + val subPipelineInfos = pipelineInfoDao.listPipelineInfoByProject( + dslContext = dslContext, + projectId = projectId, + offset = offset, + limit = step, + deleteFlag = if (includeDelete) null else false, + channelCode = ChannelCode.BS + ) ?: emptyList() + if (subPipelineInfos.isEmpty()) { + break + } + pipelineInfos.addAll(subPipelineInfos) + offset += step + hasNext = subPipelineInfos.size == step + } + pipelineInfos + } ?: emptyList() + } + + fun bulkAdd(userId: String, projectId: String, bulkAdd: PipelineViewBulkAdd): Boolean { + val isProjectManager = hasPermission(userId, projectId) + val viewIds = pipelineViewDao.list( + dslContext = dslContext, + projectId = projectId, + viewIds = bulkAdd.viewIds.map { HashUtil.decodeIdToLong(it) }.toSet() + ).asSequence() + .filter { it.viewType == PipelineViewType.STATIC } + .filter { + if (isProjectManager) { + it.isProject || it.createUser == userId + } else { + !it.isProject && it.createUser == userId + } + }.map { it.id }.toList() + if (viewIds.isEmpty()) { + logger.warn("bulkAdd , empty viewIds") + return false + } + val pipelineIds = pipelineInfoDao.listInfoByPipelineIds( + dslContext = dslContext, + projectId = projectId, + pipelineIds = bulkAdd.pipelineIds.toSet() + ).map { it.pipelineId } + if (pipelineIds.isEmpty()) { + logger.warn("bulkAdd , empty pipelineIds") + return false + } + PipelineViewGroupLock(redisOperation, projectId).lockAround { + for (viewId in viewIds) { + val existPipelineIds = + pipelineViewGroupDao.listByViewId(dslContext, projectId, viewId).map { it.pipelineId }.toSet() + pipelineIds.filterNot { existPipelineIds.contains(it) }.forEach { + try { + pipelineViewGroupDao.create( + dslContext = dslContext, + projectId = projectId, + pipelineId = it, + viewId = viewId, + userId = userId + ) + } catch (e: Exception) { + logger.info("bulkAdd , ignore exception, viewId:$viewId , pipelineId:$it") + } + } + } + } + return true + } + + fun bulkRemove(userId: String, projectId: String, bulkRemove: PipelineViewBulkRemove): Boolean { + val viewId = HashUtil.decodeIdToLong(bulkRemove.viewId) + val view = pipelineViewDao.get( + dslContext = dslContext, + projectId = projectId, + viewId = viewId + ) ?: return false + val isProjectManager = hasPermission(userId, projectId) + if (isProjectManager && !view.isProject && view.createUser != userId) { + logger.warn("bulkRemove , $userId is ProjectManager , but can`t remove other view") + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_VIEW_GROUP_NO_PERMISSION, + defaultMessage = "user:$userId has no permission to edit view group, project:$projectId" + ) + } + if (!isProjectManager && (view.isProject || view.createUser != userId)) { + logger.warn("bulkRemove , $userId isn`t ProjectManager , just can remove self view") + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_VIEW_GROUP_NO_PERMISSION, + defaultMessage = "user:$userId has no permission to edit view group, project:$projectId" + ) + } + pipelineViewGroupDao.batchRemove(dslContext, projectId, viewId, bulkRemove.pipelineIds) + return true + } + + fun hasPermission(userId: String, projectId: String) = + client.get(ServiceProjectAuthResource::class) + .checkManager(clientTokenService.getSystemToken(null)!!, userId, projectId).data ?: false + + fun listView(userId: String, projectId: String, projected: Boolean?, viewType: Int?): List { + val views = pipelineViewDao.list(dslContext, userId, projectId, projected, viewType) + val countByViewId = pipelineViewGroupDao.countByViewId(dslContext, projectId, views.map { it.id }) + // 确保数据都初始化一下 + views.filter { it.viewType == PipelineViewType.DYNAMIC } + .forEach { initDynamicViewGroup(it, userId, dslContext) } + val summaries = sortViews2Summary(projectId, userId, views, countByViewId) + if (projected != false) { + val classifiedPipelineIds = getClassifiedPipelineIds(projectId) + val unclassifiedCount = + pipelineInfoDao.countExcludePipelineIds(dslContext, projectId, classifiedPipelineIds, ChannelCode.BS) + summaries.add( + 0, PipelineNewViewSummary( + id = PIPELINE_VIEW_UNCLASSIFIED, + projectId = projectId, + name = "未分组", + projected = true, + createTime = LocalDateTime.now().timestamp(), + updateTime = LocalDateTime.now().timestamp(), + creator = "admin", + top = false, + viewType = PipelineViewType.UNCLASSIFIED, + pipelineCount = unclassifiedCount + ) + ) + } + return summaries + } + + fun listViewByPipelineId(userId: String, projectId: String, pipelineId: String): List { + val viewGroupRecords = pipelineViewGroupDao.listByPipelineId(dslContext, projectId, pipelineId) + val viewRecords = pipelineViewDao.list(dslContext, projectId, viewGroupRecords.map { it.viewId }.toSet()) + return viewRecords.filter { it.isProject || it.createUser == userId }.map { + PipelineNewViewSummary( + id = HashUtil.encodeLongId(it.id), + projectId = it.projectId, + name = it.name, + projected = it.isProject, + createTime = it.createTime.timestamp(), + updateTime = it.updateTime.timestamp(), + creator = it.createUser, + viewType = it.viewType, + pipelineCount = 0 + ) + } + } + + private fun sortViews2Summary( + projectId: String, + userId: String, + views: List, + countByViewId: Map + ): MutableList { + var score = 1 + val viewScoreMap = pipelineViewTopDao.list(dslContext, projectId, userId).associate { it.viewId to score++ } + + return views.sortedBy { + viewScoreMap[it.id] ?: Int.MAX_VALUE + }.map { + PipelineNewViewSummary( + id = HashUtil.encodeLongId(it.id), + projectId = it.projectId, + name = it.name, + projected = it.isProject, + createTime = it.createTime.timestamp(), + updateTime = it.updateTime.timestamp(), + creator = it.createUser, + top = viewScoreMap.containsKey(it.id), + viewType = it.viewType, + pipelineCount = countByViewId[it.id] ?: 0 + ) + }.toMutableList() + } + + private fun pipelineRecord2Info(record: TPipelineInfoRecord): PipelineViewPreview.PipelineInfo { + return PipelineViewPreview.PipelineInfo( + pipelineId = record.pipelineId, + pipelineName = record.pipelineName, + delete = record.delete + ) + } + + fun pipelineCount(userId: String, projectId: String, viewId: String): PipelineViewPipelineCount { + val viewGroups = pipelineViewGroupDao.listByViewId(dslContext, projectId, HashUtil.decodeIdToLong(viewId)) + if (viewGroups.isEmpty()) { + return PipelineViewPipelineCount.DEFAULT + } + val pipelineInfos = pipelineInfoDao.listInfoByPipelineIds( + dslContext, + projectId, + viewGroups.map { it.pipelineId }.toSet(), + false + ) + val deleteCount = pipelineInfos.count { it.delete } + return PipelineViewPipelineCount( + normalCount = pipelineInfos.size - deleteCount, + deleteCount = deleteCount + ) + } + + fun initAllView() { + val dynamicProjectIds = pipelineViewDao.listDynamicProjectId(dslContext) + logger.info("dynamicProjectIds : $dynamicProjectIds") + dynamicProjectIds.forEach { projectId -> + pipelineViewDao.listDynamicViewByProjectId(dslContext, projectId).forEach { view -> + redisOperation.delete(firstInitMark(view.projectId, view.id)) + logger.info("init start , ${view.projectId} , ${view.id}") + initDynamicViewGroup(view, "admin") + logger.info("init finish , ${view.projectId} , ${view.id}") + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(PipelineViewGroupService::class.java) + } +} diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/view/PipelineViewService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/view/PipelineViewService.kt index edcd39d84b5..ab16d6fe571 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/view/PipelineViewService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/view/PipelineViewService.kt @@ -36,31 +36,40 @@ import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.timestamp import com.tencent.devops.common.client.Client import com.tencent.devops.common.service.utils.MessageCodeUtil +import com.tencent.devops.model.process.tables.records.TPipelineInfoRecord import com.tencent.devops.model.process.tables.records.TPipelineViewRecord +import com.tencent.devops.process.constant.PipelineViewType import com.tencent.devops.process.constant.ProcessMessageCode import com.tencent.devops.process.dao.PipelineViewUserLastViewDao import com.tencent.devops.process.dao.PipelineViewUserSettingsDao +import com.tencent.devops.process.dao.label.PipelineGroupDao +import com.tencent.devops.process.dao.label.PipelineLabelDao +import com.tencent.devops.process.dao.label.PipelineLabelPipelineDao import com.tencent.devops.process.dao.label.PipelineViewDao +import com.tencent.devops.process.dao.label.PipelineViewTopDao +import com.tencent.devops.process.engine.dao.PipelineInfoDao import com.tencent.devops.process.permission.PipelinePermissionService import com.tencent.devops.process.pojo.classify.PipelineNewView -import com.tencent.devops.process.pojo.classify.PipelineNewViewCreate import com.tencent.devops.process.pojo.classify.PipelineNewViewSummary -import com.tencent.devops.process.pojo.classify.PipelineNewViewUpdate import com.tencent.devops.process.pojo.classify.PipelineViewClassify import com.tencent.devops.process.pojo.classify.PipelineViewFilter import com.tencent.devops.process.pojo.classify.PipelineViewFilterByCreator import com.tencent.devops.process.pojo.classify.PipelineViewFilterByLabel import com.tencent.devops.process.pojo.classify.PipelineViewFilterByName +import com.tencent.devops.process.pojo.classify.PipelineViewForm +import com.tencent.devops.process.pojo.classify.PipelineViewHitFilters import com.tencent.devops.process.pojo.classify.PipelineViewIdAndName +import com.tencent.devops.process.pojo.classify.PipelineViewMatchDynamic import com.tencent.devops.process.pojo.classify.PipelineViewSettings import com.tencent.devops.process.pojo.classify.enums.Condition import com.tencent.devops.process.pojo.classify.enums.Logic +import com.tencent.devops.process.service.label.PipelineGroupService import com.tencent.devops.process.utils.PIPELINE_VIEW_ALL_PIPELINES import com.tencent.devops.process.utils.PIPELINE_VIEW_FAVORITE_PIPELINES import com.tencent.devops.process.utils.PIPELINE_VIEW_MY_PIPELINES import com.tencent.devops.project.api.service.ServiceAllocIdResource +import org.apache.commons.lang3.StringUtils import org.jooq.DSLContext -import org.jooq.impl.DSL import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.dao.DuplicateKeyException @@ -72,12 +81,17 @@ class PipelineViewService @Autowired constructor( private val dslContext: DSLContext, private val objectMapper: ObjectMapper, private val pipelineViewDao: PipelineViewDao, + private val pipelineInfoDao: PipelineInfoDao, + private val pipelineLabelDao: PipelineLabelDao, + private val pipelineGroupDao: PipelineGroupDao, + private val pipelineLabelPipelineDao: PipelineLabelPipelineDao, + private val pipelineViewTopDao: PipelineViewTopDao, private val pipelineViewUserSettingDao: PipelineViewUserSettingsDao, private val pipelineViewLastViewDao: PipelineViewUserLastViewDao, private val pipelinePermissionService: PipelinePermissionService, + private val pipelineGroupService: PipelineGroupService, private val client: Client ) { - fun addUsingView(userId: String, projectId: String, viewId: String) { pipelineViewLastViewDao.save( dslContext = dslContext, @@ -275,7 +289,7 @@ class PipelineViewService @Autowired constructor( } fun getViews(userId: String, projectId: String): List { - val views = pipelineViewDao.listProjectOrUser( + val views = pipelineViewDao.listAll( dslContext = dslContext, projectId = projectId, isProject = true, @@ -290,55 +304,40 @@ class PipelineViewService @Autowired constructor( projected = it.isProject, createTime = it.createTime.timestamp(), updateTime = it.updateTime.timestamp(), - creator = it.createUser + creator = it.createUser, + viewType = it.viewType, + pipelineCount = 0 ) } } - fun getView(userId: String, projectId: String, viewId: String): PipelineNewView { - val viewRecord = pipelineViewDao.get(dslContext = dslContext, projectId = projectId, viewId = decode(viewId)) - ?: throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_PIPELINE_VIEW_NOT_FOUND, - params = arrayOf(viewId) - ) - - val filters = - getFilters( - filterByName = viewRecord.filterByPipeineName, - filterByCreator = viewRecord.filterByCreator, - filters = viewRecord.filters - ) - - return PipelineNewView( - id = encode(viewRecord.id), - projectId = viewRecord.projectId, - name = viewRecord.name, - projected = viewRecord.isProject, - createTime = viewRecord.createTime.timestamp(), - updateTime = viewRecord.updateTime.timestamp(), - creator = viewRecord.createUser, - logic = Logic.valueOf(viewRecord.logic), - filters = filters - ) - } - - fun addView(userId: String, projectId: String, pipelineView: PipelineNewViewCreate): String { + fun addView( + userId: String, + projectId: String, + pipelineView: PipelineViewForm, + context: DSLContext? = null + ): Long { try { - return dslContext.transactionResult { configuration -> - val context = DSL.using(configuration) - val viewId = pipelineViewDao.create( - dslContext = context, - projectId = projectId, - name = pipelineView.name, - logic = pipelineView.logic.name, - isProject = pipelineView.projected, - filters = objectMapper.writerFor(object : - TypeReference>() {}).writeValueAsString(pipelineView.filters), - userId = userId, - id = client.get(ServiceAllocIdResource::class).generateSegmentId("PIPELINE_VIEW").data - ) - encode(viewId) + pipelineView.name = pipelineView.name.trim() + checkForUpset(context, projectId, userId, pipelineView, true) + val filters = if (pipelineView.viewType == PipelineViewType.DYNAMIC) { + objectMapper.writerFor(object : + TypeReference>() {}).writeValueAsString(pipelineView.filters) + } else { + "" } + val logic = if (pipelineView.viewType == PipelineViewType.DYNAMIC) pipelineView.logic.name else "" + return pipelineViewDao.create( + dslContext = context ?: dslContext, + projectId = projectId, + name = pipelineView.name, + logic = logic, + isProject = pipelineView.projected, + filters = filters, + userId = userId, + id = client.get(ServiceAllocIdResource::class).generateSegmentId("PIPELINE_VIEW").data, + viewType = pipelineView.viewType + ) } catch (t: DuplicateKeyException) { logger.warn("Fail to create the pipeline $pipelineView by userId") throw throw ErrorCodeException( @@ -348,66 +347,98 @@ class PipelineViewService @Autowired constructor( } } - fun deleteView(userId: String, projectId: String, viewId: String): Boolean { - val id = decode(viewId) - val viewRecord = pipelineViewDao.get(dslContext, projectId, decode(viewId)) - ?: throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_PIPELINE_VIEW_NOT_FOUND, - params = arrayOf(viewId) - ) - val isUserManager = isUserManager(userId, projectId) + fun deleteView(userId: String, projectId: String, viewId: Long, context: DSLContext? = null): Boolean { + return pipelineViewDao.delete(context ?: dslContext, projectId, viewId) + } - if (!(userId == viewRecord.createUser || (viewRecord.isProject && isUserManager))) { - throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_DEL_PIPELINE_VIEW_NO_PERM, - params = arrayOf(userId, viewId) + fun updateView( + userId: String, + projectId: String, + viewId: Long, + pipelineView: PipelineViewForm, + context: DSLContext? = null + ): Boolean { + try { + checkForUpset(context, projectId, userId, pipelineView, false, viewId) + return pipelineViewDao.update( + dslContext = context ?: dslContext, + projectId = projectId, + viewId = viewId, + name = pipelineView.name, + logic = pipelineView.logic.name, + isProject = pipelineView.projected, + filters = objectMapper.writerFor(object : + TypeReference>() {}).writeValueAsString( + pipelineView.filters + ), + viewType = pipelineView.viewType + ) + } catch (t: DuplicateKeyException) { + logger.warn("Fail to update the pipeline $pipelineView by userId") + throw throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_VIEW_HAD_EXISTS, + params = arrayOf(pipelineView.name) ) - } - - return dslContext.transactionResult { configuration -> - val context = DSL.using(configuration) - pipelineViewDao.delete(context, projectId, id) } } - fun updateView(userId: String, projectId: String, viewId: String, pipelineView: PipelineNewViewUpdate): Boolean { - val id = decode(viewId) - val viewRecord = pipelineViewDao.get(dslContext = dslContext, projectId = projectId, viewId = decode(viewId)) - ?: throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_PIPELINE_VIEW_NOT_FOUND, - params = arrayOf(viewId) - ) - val isUserManager = isUserManager(userId = userId, projectId = projectId) - - if (!(userId == viewRecord.createUser || (viewRecord.isProject && isUserManager))) { + private fun checkForUpset( + context: DSLContext?, + projectId: String, + userId: String, + pipelineView: PipelineViewForm, + isCreate: Boolean, + viewId: Long? = null + ) { + if (pipelineView.name.isEmpty() || pipelineView.name.length > 16) { + logger.warn("pipeline view name is illegal , user:$userId , project:$projectId") throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_EDIT_PIPELINE_VIEW_NO_PERM, - params = arrayOf(userId, viewId) + errorCode = ProcessMessageCode.ERROR_VIEW_NAME_ILLEGAL, + defaultMessage = "pipeline group name is illegal , the length is limited to 1~16" ) } + if (isCreate) { + val countForLimit = pipelineViewDao.countForLimit( + dslContext = context ?: dslContext, + projectId = projectId, + isProject = pipelineView.projected, + userId = userId + ) + val limit = if (pipelineView.projected) PROJECT_VIEW_LIMIT else PERSONAL_VIEW_LIMIT - try { - return dslContext.transactionResult { configuration -> - val context = DSL.using(configuration) - val result = pipelineViewDao.update( - dslContext = context, - projectId = projectId, - viewId = id, - name = pipelineView.name, - logic = pipelineView.logic.name, - isProject = pipelineView.projected, - filters = objectMapper.writerFor(object : - TypeReference>() {}).writeValueAsString( - pipelineView.filters - ) + if (countForLimit + 1 >= limit) { + logger.warn("exceed the limit for create , project:$projectId , user:$userId , view:$pipelineView") + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_VIEW_EXCEED_THE_LIMIT, + defaultMessage = "exceed the limit for create , the limit is : $limit" ) - result } - } catch (t: DuplicateKeyException) { - logger.warn("Fail to update the pipeline $pipelineView by userId") - throw throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_PIPELINE_VIEW_HAD_EXISTS, - params = arrayOf(pipelineView.name) + } + val excludeIds = viewId?.let { setOf(viewId) } ?: emptySet() + val hasSameName = if (pipelineView.projected) { + pipelineViewDao.countByName( + dslContext = context ?: dslContext, + projectId = projectId, + name = pipelineView.name, + isProject = true, + excludeIds = excludeIds + ) > 0 + } else { + pipelineViewDao.countByName( + dslContext = context ?: dslContext, + projectId = projectId, + name = pipelineView.name, + creator = userId, + isProject = false, + excludeIds = excludeIds + ) > 0 + } + + if (hasSameName) { + logger.warn("duplicate name , project:$projectId , user:$userId , view:$pipelineView") + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_VIEW_DUPLICATE_NAME, + defaultMessage = "view name is duplicate , name:${pipelineView.name}" ) } } @@ -422,9 +453,11 @@ class PipelineViewService @Autowired constructor( is PipelineViewFilterByName -> { filterByNames.add(filter) } + is PipelineViewFilterByCreator -> { filterByCreators.add(filter) } + is PipelineViewFilterByLabel -> { filterByLabels.add(filter) } @@ -434,7 +467,40 @@ class PipelineViewService @Autowired constructor( return Triple(first = filterByNames, second = filterByCreators, third = filterByLabels) } - private fun getFilters( + fun matchView( + pipelineView: TPipelineViewRecord, + pipelineInfo: TPipelineInfoRecord + ): Boolean { + val filters = getFilters( + filterByName = pipelineView.filterByPipeineName, + filterByCreator = pipelineView.filterByCreator, + filters = pipelineView.filters + ) + for (filter in filters) { + val match = if (filter is PipelineViewFilterByName) { + StringUtils.containsIgnoreCase(pipelineInfo.pipelineName, filter.pipelineName) + } else if (filter is PipelineViewFilterByCreator) { + filter.userIds.contains(pipelineInfo.creator) + } else if (filter is PipelineViewFilterByLabel) { + pipelineGroupService.getViewLabelToPipelinesMap( + pipelineInfo.projectId, + filter.labelIds + ).values.asSequence().flatten().contains(pipelineInfo.pipelineId) + } else { + continue + } + + if (pipelineView.logic == Logic.OR.name && match) { + return true + } + if (pipelineView.logic == Logic.AND.name && !match) { + return false + } + } + return pipelineView.logic == Logic.AND.name + } + + fun getFilters( filterByName: String, filterByCreator: String, filters: String? @@ -474,6 +540,7 @@ class PipelineViewService @Autowired constructor( PIPELINE_VIEW_FAVORITE_PIPELINES -> { MessageCodeUtil.getCodeLanMessage(ProcessMessageCode.FAVORITE_PIPELINES_LABEL) } + PIPELINE_VIEW_MY_PIPELINES -> MessageCodeUtil.getCodeLanMessage(ProcessMessageCode.MY_PIPELINES_LABEL) PIPELINE_VIEW_ALL_PIPELINES -> MessageCodeUtil.getCodeLanMessage(ProcessMessageCode.ALL_PIPELINES_LABEL) else -> throw ErrorCodeException( @@ -486,9 +553,164 @@ class PipelineViewService @Autowired constructor( private fun decode(id: String) = HashUtil.decodeIdToLong(id) + fun topView(userId: String, projectId: String, viewId: String, enabled: Boolean): Boolean { + val viewIdLong = decode(viewId) + val view = pipelineViewDao.get(dslContext, projectId, viewIdLong) ?: return false + if (!view.isProject && view.createUser != userId) { + logger.info("top view failed because no permission") + return false + } + if (enabled) { + pipelineViewTopDao.add( + dslContext = dslContext, + projectId = projectId, + viewId = viewIdLong, + userId = userId + ) + } else { + pipelineViewTopDao.remove( + dslContext = dslContext, + projectId = projectId, + viewId = viewIdLong, + userId = userId + ) + } + return true + } + + fun getHitFilters(userId: String, projectId: String, pipelineId: String, viewId: String): PipelineViewHitFilters { + val pipelineView = pipelineViewDao.get(dslContext, projectId, decode(viewId)) + if (null == pipelineView || pipelineView.viewType == PipelineViewType.STATIC) { + return PipelineViewHitFilters.EMPTY + } + val pipelineInfo = pipelineInfoDao.getPipelineId(dslContext, projectId, pipelineId) + ?: return PipelineViewHitFilters.EMPTY + + val filters = getFilters( + filterByName = pipelineView.filterByPipeineName, + filterByCreator = pipelineView.filterByCreator, + filters = pipelineView.filters + ) + val hitFilters = PipelineViewHitFilters(filters = mutableListOf(), logic = pipelineView.logic) + + for (filter in filters) { + if (filter is PipelineViewFilterByName) { + hitFilters.filters.add( + PipelineViewHitFilters.FilterInfo( + key = "流水线名称", + hits = mutableListOf( + PipelineViewHitFilters.FilterInfo.Hit( + hit = StringUtils.containsIgnoreCase(pipelineInfo.pipelineName, filter.pipelineName), + value = filter.pipelineName + ) + ) + ) + ) + } else if (filter is PipelineViewFilterByCreator) { + filter.userIds.forEach { + hitFilters.filters.add( + PipelineViewHitFilters.FilterInfo( + key = "创建人", + hits = mutableListOf( + PipelineViewHitFilters.FilterInfo.Hit( + hit = it == pipelineInfo.creator, + value = it + ) + ) + ) + ) + } + } else if (filter is PipelineViewFilterByLabel) { + val group = pipelineGroupDao.get(dslContext, decode(filter.groupId)) ?: continue + val labels = + pipelineLabelDao.getByIds(dslContext, projectId, filter.labelIds.map { decode(it) }.toSet()) + val labelIds = pipelineLabelPipelineDao.listLabels(dslContext, projectId, pipelineId).map { it.labelId } + labels.forEach { + hitFilters.filters.add( + PipelineViewHitFilters.FilterInfo( + key = group.name, + hits = mutableListOf( + PipelineViewHitFilters.FilterInfo.Hit( + hit = labelIds.contains(it.id), + value = it.name + ) + ) + ) + ) + } + } else { + continue + } + } + return hitFilters + } + + fun matchDynamicView( + userId: String, + projectId: String, + pipelineViewMatchDynamic: PipelineViewMatchDynamic + ): List { + val viewList = pipelineViewDao.list(dslContext, projectId) + val labelGroupMap = pipelineViewMatchDynamic.labels.associate { it.groupId to it.labelIds.toSet() } + val result = mutableListOf() + for (view in viewList) { + if (!view.isProject && view.createUser != userId) { + continue + } + if (view.viewType == PipelineViewType.STATIC) { + continue + } + val filters = getFilters(view.filterByPipeineName, view.filterByCreator, view.filters) + var isMatch = view.logic == Logic.AND.name + for (filter in filters) { + val match = if (filter is PipelineViewFilterByName) { + StringUtils.containsIgnoreCase(pipelineViewMatchDynamic.pipelineName, filter.pipelineName) + } else if (filter is PipelineViewFilterByCreator) { + filter.userIds.contains(userId) + } else if (filter is PipelineViewFilterByLabel) { + val newLabels = labelGroupMap[filter.groupId] + if (newLabels != null) { + val oldLabels = filter.labelIds.toMutableSet() + oldLabels.retainAll(newLabels) + oldLabels.isNotEmpty() + } else { + false + } + } else { + continue + } + + if (view.logic == Logic.OR.name && match) { + isMatch = true + break + } + if (view.logic == Logic.AND.name && !match) { + isMatch = false + break + } + } + if (isMatch) { + result.add(encode(view.id)) + } + } + return result + } + + fun viewName2viewId( + projectId: String, + name: String, + isProject: Boolean + ): String? { + return pipelineViewDao.fetchAnyByName( + dslContext = dslContext, projectId = projectId, name = name, isProject = isProject + )?.id?.let { encode(it) } + } + companion object { private val logger = LoggerFactory.getLogger(PipelineViewService::class.java) private val SYSTEM_VIEW_ID_LIST = listOf(PIPELINE_VIEW_FAVORITE_PIPELINES, PIPELINE_VIEW_MY_PIPELINES, PIPELINE_VIEW_ALL_PIPELINES) + private const val PROJECT_VIEW_LIMIT = 200 + private const val PERSONAL_VIEW_LIMIT = 100 } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/view/lock/PipelineViewGroupLock.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/view/lock/PipelineViewGroupLock.kt new file mode 100644 index 00000000000..1ef21abc110 --- /dev/null +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/view/lock/PipelineViewGroupLock.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.service.view.lock + +import com.tencent.devops.common.redis.RedisLock +import com.tencent.devops.common.redis.RedisOperation + +class PipelineViewGroupLock(redisOperation: RedisOperation, projectId: String) : + RedisLock( + redisOperation = redisOperation, + lockKey = "p:view:group:$projectId", + expiredTimeInSeconds = 10L + ) diff --git a/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/service/view/PipelineViewGroupServiceTest.kt b/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/service/view/PipelineViewGroupServiceTest.kt new file mode 100644 index 00000000000..bd5d1fbdfd0 --- /dev/null +++ b/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/service/view/PipelineViewGroupServiceTest.kt @@ -0,0 +1,961 @@ +package com.tencent.devops.process.service.view + +import com.tencent.devops.auth.api.service.ServiceProjectAuthResource +import com.tencent.devops.common.api.exception.ErrorCodeException +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.api.util.HashUtil +import com.tencent.devops.common.client.ClientTokenService +import com.tencent.devops.common.test.BkCiAbstractTest +import com.tencent.devops.model.process.Tables.T_PIPELINE_INFO +import com.tencent.devops.model.process.Tables.T_PIPELINE_VIEW +import com.tencent.devops.model.process.Tables.T_PIPELINE_VIEW_GROUP +import com.tencent.devops.model.process.tables.records.TPipelineInfoRecord +import com.tencent.devops.model.process.tables.records.TPipelineViewGroupRecord +import com.tencent.devops.model.process.tables.records.TPipelineViewRecord +import com.tencent.devops.model.process.tables.records.TPipelineViewTopRecord +import com.tencent.devops.process.constant.PipelineViewType +import com.tencent.devops.process.constant.ProcessMessageCode +import com.tencent.devops.process.dao.label.PipelineViewDao +import com.tencent.devops.process.dao.label.PipelineViewGroupDao +import com.tencent.devops.process.dao.label.PipelineViewTopDao +import com.tencent.devops.process.engine.dao.PipelineInfoDao +import com.tencent.devops.process.permission.PipelinePermissionService +import com.tencent.devops.process.pojo.classify.PipelineNewViewSummary +import com.tencent.devops.process.pojo.classify.PipelineViewBulkAdd +import com.tencent.devops.process.pojo.classify.PipelineViewBulkRemove +import com.tencent.devops.process.pojo.classify.PipelineViewDict +import com.tencent.devops.process.pojo.classify.PipelineViewForm +import com.tencent.devops.process.pojo.classify.PipelineViewPipelineCount +import com.tencent.devops.process.pojo.classify.PipelineViewPreview +import com.tencent.devops.process.utils.PIPELINE_VIEW_UNCLASSIFIED +import io.mockk.every +import io.mockk.justRun +import io.mockk.mockk +import io.mockk.mockkObject +import io.mockk.spyk +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test +import java.time.LocalDateTime + +class PipelineViewGroupServiceTest : BkCiAbstractTest() { + private val pipelineViewService: PipelineViewService = mockk() + private val pipelinePermissionService: PipelinePermissionService = mockk() + private val pipelineViewDao: PipelineViewDao = mockk() + private val pipelineViewGroupDao: PipelineViewGroupDao = mockk() + private val pipelineViewTopDao: PipelineViewTopDao = mockk() + private val pipelineInfoDao: PipelineInfoDao = mockk() + private val clientTokenService: ClientTokenService = mockk() + + private val self: PipelineViewGroupService = spyk( + PipelineViewGroupService( + pipelineViewService = pipelineViewService, + pipelineViewDao = pipelineViewDao, + pipelineViewGroupDao = pipelineViewGroupDao, + pipelineViewTopDao = pipelineViewTopDao, + pipelineInfoDao = pipelineInfoDao, + dslContext = dslContext, + redisOperation = redisOperation, + objectMapper = objectMapper, + client = client, + clientTokenService = clientTokenService + ), + recordPrivateCalls = true + ) + + private val now = LocalDateTime.now() + + private val pvg = TPipelineViewGroupRecord( + /*id*/1, + /*projectId*/"test", + /*viewId*/1, + /*pipelineId*/"p-test", + /*createTime*/now, + /*creator*/"test" + ) + + private val pv = TPipelineViewRecord( + /*id*/1, + /*projectId*/"test", + /*name*/"test", + /*filterByPipeineName*/"", + /*filterByCreator*/"", + /*createTime*/now, + /*updateTime*/ now, + /*createUser*/"test", + /*isProject*/true, + /*logic*/"AND", + /*filters*/"", + /*viewType*/PipelineViewType.DYNAMIC + ) + + private val pi = TPipelineInfoRecord( + "p-test", // setPipelineId(pipelineId); + "test", // setProjectId(projectId); + "test", // setPipelineName(pipelineName); + "test", // setPipelineDesc(pipelineDesc); + 1, // setVersion(version); + now, // setCreateTime(createTime); + "test", // setCreator(creator); + now, // setUpdateTime(updateTime); + "test", // setLastModifyUser(lastModifyUser); + "test", // setChannel(channel); + 1, // setManualStartup(manualStartup); + 1, // setElementSkip(elementSkip); + 1, // setTaskCount(taskCount); + false, // setDelete(delete); + 1, // setId(id); + "test", // setPipelineNamePinyin(pipelineNamePinyin); + now // setLatestStartTime(latestStartTime); + ) + + private val pipelineViewForm = PipelineViewForm( + id = "test", + name = "test", + projected = true + ) + + @BeforeEach + fun mockViewId() { + mockkObject(HashUtil) + every { HashUtil.decodeIdToLong(any()) } returns 1L + } + + @Nested + inner class GetViewNameMap { + @Test + @DisplayName("ViewGroup列表为空") + fun test_1() { + every { pipelineViewGroupDao.listByPipelineIds(any(), any(), any()) } returns emptyList() + Assertions.assertTrue(self.getViewNameMap("", mutableSetOf()).isEmpty()) + } + + @Test + @DisplayName("View列表为空") + fun test_2() { + every { pipelineViewGroupDao.listByPipelineIds(any(), any(), any()) } returns listOf(pvg) + + every { + pipelineViewDao.list( + dslContext = any(), + projectId = any(), + viewIds = any() + ) + } returns dslContext.mockResult(T_PIPELINE_VIEW) + + Assertions.assertTrue(self.getViewNameMap("", mutableSetOf()).isEmpty()) + } + + @Test + @DisplayName("正常数据") + fun test_3() { + every { pipelineViewGroupDao.listByPipelineIds(any(), any(), any()) } returns listOf(pvg) + + every { + pipelineViewDao.list( + dslContext = any(), + projectId = any(), + viewIds = any() + ) + } returns dslContext.mockResult(T_PIPELINE_VIEW, pv) + + val viewNameMap = self.getViewNameMap("", mutableSetOf()) + Assertions.assertEquals(viewNameMap[pvg.pipelineId]?.get(0), pv.name) + } + } + + @Nested + inner class ADDViewGroup { + @BeforeEach + fun beforeEach() { + every { pipelinePermissionService.checkProjectManager("true", any()) } returns true + every { pipelinePermissionService.checkProjectManager("false", any()) } returns false + justRun { self["checkPermission"](any() as String, any() as String, any() as Boolean, any() as String?) } + } + + @Test + @DisplayName("项目流水线组 & 有权限") + fun test_1() { + val projectId = "test" + val userId = "true" + val viewId = 1L + every { pipelineViewService.addView(any(), any(), any(), any()) } returns viewId + justRun { + self["initViewGroup"]( + anyDslContext(), + pipelineViewForm, + projectId, + viewId, + userId + ) + } + Assertions.assertDoesNotThrow { + self.addViewGroup( + projectId, + userId, + pipelineViewForm + ) + } + } + } + + @Nested + inner class UpdateViewGroup { + @BeforeEach + fun beforeEach() { + justRun { self["checkPermission"](any() as String, any() as String, any() as Boolean, any() as String?) } + } + + @Test + @DisplayName("获取不到View") + fun test_1() { + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns null + + try { + self.updateViewGroup("test", "test", "test", pipelineViewForm) + } catch (e: Throwable) { + Assertions.assertThrows(ErrorCodeException::class.java) { throw e } + Assertions.assertEquals( + (e as ErrorCodeException).errorCode, + ProcessMessageCode.ERROR_PIPELINE_VIEW_NOT_FOUND + ) + } + } + + @Test + @DisplayName("有view & 权限通过 & 两个组的范围不一致") + fun test_2() { + val pipelineViewFormCopy = pipelineViewForm.copy(projected = true) + val pvCopy = pv.copy() + pvCopy.isProject = false + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns pvCopy + try { + self.updateViewGroup("test", "test", "test", pipelineViewFormCopy) + } catch (e: Throwable) { + Assertions.assertThrows(ErrorCodeException::class.java) { throw e } + Assertions.assertEquals( + (e as ErrorCodeException).errorCode, + ProcessMessageCode.ERROR_VIEW_GROUP_IS_PROJECT_NO_SAME + ) + } + } + + @Test + @DisplayName("有view & 权限通过 & 范围一致 & 正常更新") + fun test_3() { + val pipelineViewFormCopy = pipelineViewForm.copy(projected = true) + val pvCopy = pv.copy() + pvCopy.isProject = true + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns pvCopy + every { pipelineViewService.updateView(any(), any(), any(), any(), anyDslContext()) } returns true + justRun { pipelineViewGroupDao.remove(anyDslContext(), any(), any()) } + every { self["firstInitMark"](any() as String, any() as Long) } returns "test" + justRun { redisOperation.delete(any() as String) } + justRun { + self["initViewGroup"]( + anyDslContext(), + pipelineViewFormCopy, + any() as String, + any() as Long, + any() as String + ) + } + Assertions.assertTrue(self.updateViewGroup("test", "test", "test", pipelineViewFormCopy)) + } + } + + @Nested + inner class GetView { + @Test + @DisplayName("View为空") + fun test_1() { + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns null + try { + self.getView("test", "test", "test") + } catch (e: Throwable) { + Assertions.assertThrows(ErrorCodeException::class.java) { throw e } + Assertions.assertEquals( + (e as ErrorCodeException).errorCode, + ProcessMessageCode.ERROR_PIPELINE_VIEW_NOT_FOUND + ) + } + } + + @Test + @DisplayName("View不为空 , filter返回空列表 , ViewGroup返回空列表 , 正常返回") + fun test_2() { + val viewId = "test" + val projectId = "test" + val userId = "test" + + val pvCopy = pv.copy() + pvCopy.projectId = projectId + pvCopy.createUser = userId + + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns pv + every { pipelineViewService.getFilters(any(), any(), any()) } returns emptyList() + every { pipelineViewGroupDao.listByViewId(anyDslContext(), any(), any()) } returns emptyList() + self.getView(userId, projectId, viewId).let { + Assertions.assertTrue(it.filters.isEmpty()) + Assertions.assertTrue(it.pipelineIds.isEmpty()) + Assertions.assertEquals(it.id, viewId) + Assertions.assertEquals(it.projectId, projectId) + Assertions.assertEquals(it.creator, userId) + } + } + } + + @Nested + inner class DeleteViewGroup { + @BeforeEach + fun beforeEach() { + justRun { self["checkPermission"](any() as String, any() as String, any() as Boolean, any() as String?) } + } + + @Test + @DisplayName("获取不到view") + fun test_1() { + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns null + try { + self.deleteViewGroup("test", "test", "test") + } catch (e: Throwable) { + Assertions.assertThrows(ErrorCodeException::class.java) { throw e } + Assertions.assertEquals( + (e as ErrorCodeException).errorCode, + ProcessMessageCode.ERROR_PIPELINE_VIEW_NOT_FOUND + ) + } + } + + @Test + @DisplayName("获取到view , 正常执行") + fun test_2() { + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns pv + every { pipelineViewService.deleteView(any(), any(), any()) } returns true + justRun { pipelineViewGroupDao.remove(anyDslContext(), any(), any()) } + Assertions.assertTrue(self.deleteViewGroup("test", "test", "test")) + } + } + + @Nested + inner class GetClassifiedPipelineIds { + @Test + @DisplayName("正常执行") + fun test_1() { + every { pipelineViewGroupDao.distinctPipelineIds(anyDslContext(), any(), any()) } returns listOf("test") + every { + pipelineViewDao.list( + anyDslContext(), + any() as String, + any() as Boolean + ) + } returns dslContext.mockResult( + T_PIPELINE_VIEW, pv + ) + self.getClassifiedPipelineIds("test").let { + Assertions.assertTrue(it.size == 1) + Assertions.assertTrue(it[0] == "test") + } + } + } + + @Nested + inner class ListPipelineIdsByViewIds { + @Test + @DisplayName("当ViewGroup为空列表时") + fun test_1() { + every { pipelineViewGroupDao.listByViewIds(anyDslContext(), any(), any()) } returns emptyList() + self.listPipelineIdsByViewIds("test", listOf("test")).let { + Assertions.assertTrue(it.size == 1) + Assertions.assertEquals(it[0], "##NONE##") + } + } + + @Test + @DisplayName("当ViewGroup不为空列表时") + fun test_2() { + every { pipelineViewGroupDao.listByViewIds(anyDslContext(), any(), any()) } returns listOf(pvg) + self.listPipelineIdsByViewIds("test", listOf("test")).let { + Assertions.assertTrue(it.size == 1) + Assertions.assertEquals(it[0], pvg.pipelineId) + } + } + } + + @Nested + inner class ListPipelineIdsByViewId { + @Test + @DisplayName("正常返回") + fun test_1() { + every { + self["listPipelineIdsByViewIds"]( + any() as String, + any() as List + ) + } returns emptyList() + Assertions.assertTrue(self.listPipelineIdsByViewId("test", "test").isEmpty()) + } + } + + @Nested + inner class UpdateGroupAfterPipelineCreate { + @Test + @DisplayName("正常执行 , viewGroup 数量不为0") + fun test_1() { + every { pipelineInfoDao.getPipelineId(anyDslContext(), any(), any()) } returns pi + every { pipelineViewGroupDao.countByPipelineId(anyDslContext(), any(), any()) } returns 1 + Assertions.assertDoesNotThrow { self.updateGroupAfterPipelineCreate("test", "test", "test") } + } + + @Test + @DisplayName("正常执行 , viewGroup 数量为0") + fun test_2() { + every { pipelineInfoDao.getPipelineId(anyDslContext(), any(), any()) } returns pi + every { pipelineViewGroupDao.countByPipelineId(anyDslContext(), any(), any()) } returns 0 + every { + pipelineViewDao.list(anyDslContext(), any() as String, any() as Int) + } returns dslContext.mockResult(T_PIPELINE_VIEW, pv) + every { pipelineViewService.matchView(any(), any()) } returns true + justRun { pipelineViewGroupDao.create(anyDslContext(), any(), any(), any(), any()) } + Assertions.assertDoesNotThrow { self.updateGroupAfterPipelineCreate("test", "test", "test") } + } + } + + @Nested + inner class UpdateGroupAfterPipelineUpdate { + @Test + @DisplayName("正常调用 , 没有新命中的流水线组 , 没有需要删除的流水线组") + fun test_1() { + every { pipelineInfoDao.getPipelineId(anyDslContext(), any(), any()) } returns pi + every { + pipelineViewDao.list(anyDslContext(), any() as String, any() as Int) + } returns dslContext.mockResult(T_PIPELINE_VIEW, pv) + every { pipelineViewService.matchView(any(), any()) } returns true + every { pipelineViewGroupDao.listByPipelineId(anyDslContext(), any(), any()) } returns listOf(pvg) + justRun { pipelineViewGroupDao.create(anyDslContext(), any(), any(), any(), any()) } + justRun { pipelineViewGroupDao.remove(anyDslContext(), any(), any(), any()) } + Assertions.assertDoesNotThrow { self.updateGroupAfterPipelineUpdate("test", "p-test", "test") } + } + } + + @Nested + inner class InitVIewGroup { + @Test + @DisplayName("初始化动态项目组") + fun test_1() { + val pipelineViewFormCopy = pipelineViewForm.copy(viewType = PipelineViewType.DYNAMIC) + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns pv + every { + self["initDynamicViewGroup"]( + any(), + any(), + anyDslContext() + ) + } returns emptyList() + Assertions.assertDoesNotThrow { + self.invokePrivate("initViewGroup", dslContext, pipelineViewFormCopy, "test", 1, "test") + } + } + + @Test + @DisplayName("初始化静态项目组") + fun test_2() { + val pipelineViewFormCopy = + pipelineViewForm.copy(viewType = PipelineViewType.STATIC, pipelineIds = listOf("1")) + justRun { pipelineViewGroupDao.create(anyDslContext(), any(), any(), any(), any()) } + Assertions.assertDoesNotThrow { + self.invokePrivate("initViewGroup", dslContext, pipelineViewFormCopy, "test", 1, "test") + } + } + } + + @Nested + inner class InitDynamicViewGroup { + @Test + @DisplayName("不是首次初始化") + fun test_1() { + every { redisOperation.setIfAbsent(any(), any()) } returns false + self.invokePrivate>("initDynamicViewGroup", pv, "test", dslContext).let { + Assertions.assertTrue(it!!.isEmpty()) + } + } + + @Test + @DisplayName("首次初始化") + fun test_2() { + every { redisOperation.setIfAbsent(any(), any(), any(), any()) } returns true + every { self["allPipelineInfos"](any() as String, any() as Boolean) } returns listOf(pi) + every { pipelineViewService.matchView(any(), any()) } returns true + every { pipelineViewGroupDao.create(anyDslContext(), any(), any(), any(), any()) } returns Unit + self.invokePrivate>("initDynamicViewGroup", pv, "test", dslContext).let { + Assertions.assertEquals(it!!.size, 1) + Assertions.assertEquals(it[0], "p-test") + } + } + } + + @Nested + inner class CheckPermission { + @Test + @DisplayName("项目流水线组 & 校验不通过") + fun test_1() { + every { self["hasPermission"](any() as String, any() as String) } returns false + try { + self.invokePrivate("checkPermission", "test", "test", true, "test") + } catch (e: Throwable) { + Assertions.assertThrows(ErrorCodeException::class.java) { throw e } + Assertions.assertEquals( + (e as ErrorCodeException).errorCode, + ProcessMessageCode.ERROR_VIEW_GROUP_NO_PERMISSION + ) + } + } + + @Test + @DisplayName("个人流水线组 & 校验不通过") + fun test_2() { + try { + self.invokePrivate("checkPermission", "test", "test", false, "test") + } catch (e: Throwable) { + Assertions.assertThrows(ErrorCodeException::class.java) { throw e } + Assertions.assertEquals( + (e as ErrorCodeException).errorCode, + ProcessMessageCode.ERROR_DEL_PIPELINE_VIEW_NO_PERM + ) + } + } + } + + @Nested + inner class Preview { + @Test + @DisplayName("流水线列表为空") + fun test_1() { + every { + self["allPipelineInfos"]( + any() as String, + any() as Boolean + ) + } returns emptyList() + self.preview("test", "test", pipelineViewForm).let { + Assertions.assertEquals(it, PipelineViewPreview.EMPTY) + } + } + + @Test + @DisplayName("viewId为空 , viewType为动态项目组") + fun test_2() { + val pipelineViewFormCopy = pipelineViewForm.copy(id = null, viewType = PipelineViewType.DYNAMIC) + every { + self["allPipelineInfos"]( + any() as String, + any() as Boolean + ) + } returns listOf(pi) + every { pipelineViewService.matchView(any(), any()) } returns true + self.preview("test", "test", pipelineViewFormCopy).let { + Assertions.assertTrue(it.addedPipelineInfos.size == 1) + Assertions.assertTrue(it.removedPipelineInfos.isEmpty()) + Assertions.assertTrue(it.reservePipelineInfos.isEmpty()) + } + } + + @Test + @DisplayName("viewId为空,viewType为静态项目组") + fun test_3() { + val pipelineViewFormCopy = + pipelineViewForm.copy(id = null, viewType = PipelineViewType.STATIC, pipelineIds = listOf("p-test")) + every { + self["allPipelineInfos"]( + any() as String, + any() as Boolean + ) + } returns listOf(pi) + every { pipelineViewService.matchView(any(), any()) } returns true + self.preview("test", "test", pipelineViewFormCopy).let { + Assertions.assertTrue(it.addedPipelineInfos.size == 1) + Assertions.assertTrue(it.removedPipelineInfos.isEmpty()) + Assertions.assertTrue(it.reservePipelineInfos.isEmpty()) + } + } + + @Test + @DisplayName("viewId不为空,viewType为动态项目组") + fun test_4() { + val pipelineViewFormCopy = pipelineViewForm.copy(id = "test", viewType = PipelineViewType.DYNAMIC) + every { + self["allPipelineInfos"]( + any() as String, + any() as Boolean + ) + } returns listOf(pi) + every { pipelineViewGroupDao.listByViewId(anyDslContext(), any(), any()) } returns emptyList() + every { pipelineViewService.matchView(any(), any()) } returns true + self.preview("test", "test", pipelineViewFormCopy).let { + Assertions.assertTrue(it.addedPipelineInfos.size == 1) + Assertions.assertTrue(it.removedPipelineInfos.isEmpty()) + Assertions.assertTrue(it.reservePipelineInfos.isEmpty()) + } + } + + @Test + @DisplayName("viewId不为空,viewType为静态项目组") + fun test_5() { + val pipelineViewFormCopy = + pipelineViewForm.copy(id = "test", viewType = PipelineViewType.STATIC, pipelineIds = listOf("p-test")) + every { + self["allPipelineInfos"]( + any() as String, + any() as Boolean + ) + } returns listOf(pi) + every { pipelineViewGroupDao.listByViewId(anyDslContext(), any(), any()) } returns emptyList() + every { pipelineViewService.matchView(any(), any()) } returns true + self.preview("test", "test", pipelineViewFormCopy).let { + Assertions.assertTrue(it.addedPipelineInfos.size == 1) + Assertions.assertTrue(it.removedPipelineInfos.isEmpty()) + Assertions.assertTrue(it.reservePipelineInfos.isEmpty()) + } + } + } + + @Nested + inner class Dict { + @Test + @DisplayName("ViewInfo列表 不为空 , viewInfoGroup列表不为空, pipelineInfo列表为空") + fun test_1() { + every { pipelineViewDao.list(anyDslContext(), any()) } returns dslContext.mockResult(T_PIPELINE_VIEW, pv) + every { + pipelineViewGroupDao.listByProjectId(anyDslContext(), any()) + } returns dslContext.mockResult(T_PIPELINE_VIEW_GROUP, pvg) + every { + self["allPipelineInfos"]( + any() as String, + any() as Boolean + ) + } returns emptyList() + self.dict("test", "test").let { + Assertions.assertEquals(it, PipelineViewDict.EMPTY) + } + } + + @Test + @DisplayName("ViewInfo列表 不为空 , viewInfoGroup列表不为空, pipelineInfo列表不为空") + fun test_2() { + every { pipelineViewDao.list(anyDslContext(), any()) } returns dslContext.mockResult(T_PIPELINE_VIEW, pv) + every { + pipelineViewGroupDao.listByProjectId(anyDslContext(), any()) + } returns dslContext.mockResult(T_PIPELINE_VIEW_GROUP, pvg) + every { + self["allPipelineInfos"]( + any() as String, + any() as Boolean + ) + } returns listOf(pi) + self.dict("test", "test").let { + Assertions.assertTrue(it.projectViewList.size == 2) + Assertions.assertTrue(it.personalViewList.isEmpty()) + Assertions.assertTrue( + it.projectViewList.filter { p -> p.viewId == PIPELINE_VIEW_UNCLASSIFIED }.size == 1 + ) + } + } + } + + @Nested + inner class AllPipelineInfos { + @Test + @DisplayName("PipelineInfo有数据") + fun test_1() { + every { + pipelineInfoDao.listPipelineInfoByProject(anyDslContext(), any(), any(), any(), any(), any(), any()) + } returns dslContext.mockResult(T_PIPELINE_INFO, pi) + self.invokePrivate>("allPipelineInfos", "test", false).let { + Assertions.assertEquals(it!!.size, 1) + Assertions.assertEquals(it[0].pipelineId, "p-test") + } + } + + @Test + @DisplayName("PipelineInfo无数据") + fun test_2() { + every { + pipelineInfoDao.listPipelineInfoByProject(anyDslContext(), any(), any(), any(), any(), any(), any()) + } returns dslContext.mockResult(T_PIPELINE_INFO) + self.invokePrivate>("allPipelineInfos", "test", false).let { + Assertions.assertEquals(it!!.size, 0) + } + } + } + + @Nested + inner class BulkAdd { + @BeforeEach + fun permissionFalse() { + every { self["hasPermission"]("false", any() as String) } returns false + every { self["hasPermission"]("true", any() as String) } returns true + } + + private val ba = PipelineViewBulkAdd( + pipelineIds = listOf("p-test"), + viewIds = listOf("test") + ) + + @Test + @DisplayName("ViewIds 为空") + fun test_1() { + every { + pipelineViewDao.list(anyDslContext(), any() as String, any() as Set) + } returns dslContext.mockResult(T_PIPELINE_VIEW) + self.bulkAdd("true", "test", ba).let { + Assertions.assertEquals(it, false) + } + } + + @Test + @DisplayName("项目管理员 , 为项目流水线组, 但流水线信息为空") + fun test_2() { + val pvCopy = pv.copy() + pvCopy.viewType = PipelineViewType.STATIC + pvCopy.isProject = true + pvCopy.id = 1 + every { + pipelineViewDao.list(anyDslContext(), any() as String, any() as Set) + } returns dslContext.mockResult(T_PIPELINE_VIEW, pvCopy) + every { + pipelineInfoDao.listInfoByPipelineIds(anyDslContext(), any(), any()) + } returns dslContext.mockResult(T_PIPELINE_INFO) + self.bulkAdd("true", "test", ba).let { + Assertions.assertEquals(it, false) + } + } + + @Test + @DisplayName("项目管理员 , 为项目流水线组, 流水线信息不为空 , 正常运行") + fun test_3() { + val pvCopy = pv.copy() + pvCopy.viewType = PipelineViewType.STATIC + pvCopy.isProject = true + pvCopy.id = 1 + every { + pipelineViewDao.list(anyDslContext(), any() as String, any() as Set) + } returns dslContext.mockResult(T_PIPELINE_VIEW, pvCopy) + every { + pipelineInfoDao.listInfoByPipelineIds(anyDslContext(), any(), any()) + } returns dslContext.mockResult(T_PIPELINE_INFO, pi) + every { + pipelineViewGroupDao.listByViewId(anyDslContext(), any(), any()) + } returns dslContext.mockResult(T_PIPELINE_VIEW_GROUP) + self.bulkAdd("true", "test", ba).let { + Assertions.assertEquals(it, true) + } + } + } + + @Nested + inner class BulkRemove { + private val br = PipelineViewBulkRemove( + pipelineIds = listOf("p-test"), + viewId = "test" + ) + + @BeforeEach + fun permissionFalse() { + every { self["hasPermission"]("false", any() as String) } returns false + every { self["hasPermission"]("true", any() as String) } returns true + } + + @Test + @DisplayName("view 为空") + fun test_1() { + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns null + self.bulkRemove("test", "test", br).let { + Assertions.assertEquals(it, false) + } + } + + @Test + @DisplayName("view不为空 ,静态组, 项目管理员 , 不是项目流水线组 , 创建者不是自己") + fun test_2() { + val pvCopy = pv.copy() + pvCopy.viewType = PipelineViewType.STATIC + pvCopy.isProject = false + pvCopy.createUser = "other" + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns pvCopy + try { + self.bulkRemove("true", "test", br) + } catch (e: Exception) { + Assertions.assertThrows(ErrorCodeException::class.java) { throw e } + Assertions.assertEquals( + (e as ErrorCodeException).errorCode, + ProcessMessageCode.ERROR_VIEW_GROUP_NO_PERMISSION + ) + } + } + + @Test + @DisplayName("view不为空 ,静态组, 不是项目管理员 , 是项目流水线组") + fun test_3() { + val pvCopy = pv.copy() + pvCopy.viewType = PipelineViewType.STATIC + pvCopy.isProject = true + pvCopy.createUser = "other" + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns pvCopy + try { + self.bulkRemove("false", "test", br) + } catch (e: Exception) { + Assertions.assertThrows(ErrorCodeException::class.java) { throw e } + Assertions.assertEquals( + (e as ErrorCodeException).errorCode, + ProcessMessageCode.ERROR_VIEW_GROUP_NO_PERMISSION + ) + } + } + + @Test + @DisplayName("view不为空 ,静态组, 是项目管理员 , 是项目流水线组") + fun test_4() { + val pvCopy = pv.copy() + pvCopy.viewType = PipelineViewType.STATIC + pvCopy.isProject = true + pvCopy.createUser = "other" + every { pipelineViewDao.get(anyDslContext(), any(), any()) } returns pvCopy + every { pipelineViewGroupDao.batchRemove(anyDslContext(), any(), any(), any()) } returns Unit + self.bulkRemove("true", "test", br).let { + Assertions.assertEquals(it, true) + } + } + } + + @Nested + inner class HasPermission { + @BeforeEach + fun beforeEach() { + every { clientTokenService.getSystemToken(any()) } returns "" + } + + @Test + @DisplayName("返回值测试1") + fun test_1() { + every { + client.mockGet(ServiceProjectAuthResource::class).checkManager(any(), any(), any()) + } returns Result(true) + self.hasPermission("test", "test").let { + Assertions.assertEquals(true, it) + } + } + + @Test + @DisplayName("返回值测试2") + fun test_2() { + every { + client.mockGet(ServiceProjectAuthResource::class).checkManager(any(), any(), any()) + } returns Result(false) + self.hasPermission("test", "test").let { + Assertions.assertEquals(false, it) + } + } + } + + @Nested + inner class ListView { + @Test + @DisplayName("是项目流水线组") + fun test_1() { + every { pipelineViewDao.list(anyDslContext(), any(), any(), any(), any()) } returns emptyList() + every { pipelineViewGroupDao.countByViewId(anyDslContext(), any(), any()) } returns emptyMap() + every { + self["sortViews2Summary"]( + any() as String, + any() as String, + any() as List, + any() as Map + ) + } returns mutableListOf() + self.listView("test", "test", false, PipelineViewType.DYNAMIC).let { + Assertions.assertEquals(it.size, 0) + } + } + + @Test + @DisplayName("不是项目流水线组") + fun test_2() { + every { pipelineViewDao.list(anyDslContext(), any(), any(), any(), any()) } returns emptyList() + every { pipelineViewGroupDao.countByViewId(anyDslContext(), any(), any()) } returns emptyMap() + every { + self["sortViews2Summary"]( + any() as String, + any() as String, + any() as List, + any() as Map + ) + } returns mutableListOf() + every { self["getClassifiedPipelineIds"](any() as String) } returns emptyList() + every { pipelineInfoDao.countExcludePipelineIds(anyDslContext(), any(), any(), any()) } returns 0 + self.listView("test", "test", true, PipelineViewType.DYNAMIC).let { + Assertions.assertEquals(it.size, 1) + Assertions.assertEquals(it[0].id, PIPELINE_VIEW_UNCLASSIFIED) + } + } + } + + @Nested + inner class SortViews2Summary { + @Test + @DisplayName("正常排序") + fun test_1() { + val pvCopy1 = pv.copy() + pvCopy1.id = 1 + pvCopy1.name = "test1" + + val pvCopy2 = pv.copy() + pvCopy2.id = 2 + pvCopy2.name = "test2" + + val pvt = TPipelineViewTopRecord() + pvt.viewId = pvCopy2.id + + every { pipelineViewTopDao.list(anyDslContext(), any(), any()) } returns listOf(pvt) + self.invokePrivate>( + "sortViews2Summary", "test", "test", listOf(pvCopy1, pvCopy2), emptyMap() + ).let { + Assertions.assertEquals(it!!.size, 2) + Assertions.assertEquals(it[0].name, pvCopy2.name) + } + } + } + + @Nested + inner class PipelineCount { + @Test + @DisplayName("viewGroup为空") + fun test_1() { + every { pipelineViewGroupDao.listByViewId(anyDslContext(), any(), any()) } returns emptyList() + self.pipelineCount("test", "test", "test").let { + Assertions.assertEquals(it, PipelineViewPipelineCount.DEFAULT) + } + } + + @Test + @DisplayName("viewGroup不为空") + fun test_2() { + every { pipelineViewGroupDao.listByViewId(anyDslContext(), any(), any()) } returns listOf(pvg) + every { + pipelineInfoDao.listInfoByPipelineIds( + anyDslContext(), + any(), + any(), + any() + ) + } returns dslContext.mockResult( + T_PIPELINE_INFO, pi + ) + self.pipelineCount("test", "test", "test").let { + Assertions.assertEquals(it.deleteCount, 0) + Assertions.assertEquals(it.normalCount, 1) + } + } + } +} diff --git a/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/utils/StreamDispatchUtils.kt b/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/utils/StreamDispatchUtils.kt index acb34cc3aa5..561877f5597 100644 --- a/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/utils/StreamDispatchUtils.kt +++ b/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/utils/StreamDispatchUtils.kt @@ -39,8 +39,10 @@ import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.enums.VMBaseOS import com.tencent.devops.common.pipeline.type.DispatchType import com.tencent.devops.common.pipeline.type.agent.AgentType +import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentDockerInfo import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentEnvDispatchType import com.tencent.devops.common.pipeline.type.docker.DockerDispatchType +import com.tencent.devops.common.pipeline.type.agent.Credential as thirdPartDockerCredential import com.tencent.devops.common.pipeline.type.docker.ImageType import com.tencent.devops.process.pojo.BuildTemplateAcrossInfo import com.tencent.devops.process.yaml.v2.models.Resources @@ -54,7 +56,7 @@ import org.slf4j.LoggerFactory import java.util.Base64 import javax.ws.rs.core.Response -@Suppress("NestedBlockDepth", "ComplexMethod") +@Suppress("ALL") object StreamDispatchUtils { private val logger = LoggerFactory.getLogger(StreamDispatchUtils::class.java) @@ -112,11 +114,43 @@ object StreamDispatchUtils { // 第三方构建机 if (job.runsOn.selfHosted == true) { + if (job.runsOn.container == null) { + return ThirdPartyAgentEnvDispatchType( + envProjectId = null, + envName = poolName, + workspace = workspace, + agentType = AgentType.NAME, + dockerInfo = null + ) + } + + val (image, userName, password) = parseRunsOnContainer( + client = client, + job = job, + projectCode = projectCode, + context = context, + buildTemplateAcrossInfo = buildTemplateAcrossInfo + ) + + val dockerInfo = ThirdPartyAgentDockerInfo( + image = image, + credential = if (userName.isBlank() || password.isBlank()) { + null + } else { + thirdPartDockerCredential( + user = userName, + password = password + ) + }, + envs = job.env + ) + return ThirdPartyAgentEnvDispatchType( envProjectId = null, envName = poolName, workspace = workspace, - agentType = AgentType.NAME + agentType = AgentType.NAME, + dockerInfo = dockerInfo ) } @@ -167,6 +201,50 @@ object StreamDispatchUtils { } } + /** + * 解析 jobs.runsOn.container + * @return image,username,password + */ + fun parseRunsOnContainer( + client: Client, + job: Job, + projectCode: String, + context: Map?, + buildTemplateAcrossInfo: BuildTemplateAcrossInfo? + ): Triple { + return try { + val container = YamlUtil.getObjectMapper().readValue( + JsonUtil.toJson(job.runsOn.container!!), + Container::class.java + ) + + Triple( + EnvUtils.parseEnv(container.image, context ?: mapOf()), + EnvUtils.parseEnv(container.credentials?.username, context ?: mapOf()), + EnvUtils.parseEnv(container.credentials?.password, context ?: mapOf()) + ) + } catch (e: Exception) { + val container = YamlUtil.getObjectMapper().readValue( + JsonUtil.toJson(job.runsOn.container!!), + Container2::class.java + ) + + var user = "" + var password = "" + if (!container.credentials.isNullOrEmpty()) { + val ticketsMap = getTicket(client, projectCode, container, context, buildTemplateAcrossInfo) + user = ticketsMap["v1"] as String + password = ticketsMap["v2"] as String + } + + Triple( + EnvUtils.parseEnv(container.image, context ?: mapOf()), + user, + password + ) + } + } + private fun getTicket( client: Client, projectCode: String, @@ -214,7 +292,7 @@ object StreamDispatchUtils { if (credentialResult.isNotOk() || credentialResult.data == null) { throw RuntimeException( "Fail to get the credential($credentialId) of project($projectId), " + - "because of ${credentialResult.message}" + "because of ${credentialResult.message}" ) } @@ -222,14 +300,14 @@ object StreamDispatchUtils { if (type != credential.credentialType) { throw ParamBlankException( "Fail to get the credential($credentialId) of project($projectId), " + - "expect:${type.name}, but real:${credential.credentialType.name}" + "expect:${type.name}, but real:${credential.credentialType.name}" ) } if (acrossProject && !credential.allowAcrossProject) { throw RuntimeException( "Fail to get the credential($credentialId) of project($projectId), " + - "not allow across project use" + "not allow across project use" ) } diff --git a/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/image/PoolType.kt b/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/image/PoolType.kt index 841fa543207..b64431e5a23 100644 --- a/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/image/PoolType.kt +++ b/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/models/image/PoolType.kt @@ -62,26 +62,30 @@ enum class PoolType { envProjectId = null, envName = pool.envName!!, workspace = pool.workspace, - agentType = AgentType.NAME + agentType = AgentType.NAME, + dockerInfo = null ) } else if (!pool.envId.isNullOrBlank()) { return ThirdPartyAgentEnvDispatchType( envProjectId = null, envName = pool.envId!!, workspace = pool.workspace, - agentType = AgentType.ID + agentType = AgentType.ID, + dockerInfo = null ) } else if (!pool.agentId.isNullOrBlank()) { return ThirdPartyAgentIDDispatchType( displayName = pool.agentId!!, workspace = pool.workspace, - agentType = AgentType.ID + agentType = AgentType.ID, + dockerInfo = null ) } else { return ThirdPartyAgentIDDispatchType( displayName = pool.agentName!!, workspace = pool.workspace, - agentType = AgentType.NAME + agentType = AgentType.NAME, + dockerInfo = null ) } } diff --git a/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt b/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt index 149fc79a080..d11b8500001 100644 --- a/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt +++ b/src/backend/ci/core/process/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt @@ -719,7 +719,7 @@ object ScriptYmlUtils { return null } - if (preTriggerOn.openapi != EnableType.TRUE.value || preTriggerOn.openapi != EnableType.FALSE.value) { + if (preTriggerOn.openapi != EnableType.TRUE.value && preTriggerOn.openapi != EnableType.FALSE.value) { throw YamlFormatException("not allow openapi type ${preTriggerOn.openapi}") } diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceAllocIdResource.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceAllocIdResource.kt index 8522bd57b17..07f316faf65 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceAllocIdResource.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceAllocIdResource.kt @@ -37,6 +37,7 @@ import javax.ws.rs.GET import javax.ws.rs.Path import javax.ws.rs.PathParam import javax.ws.rs.Produces +import javax.ws.rs.QueryParam import javax.ws.rs.core.MediaType @Api(tags = ["SERVICE_ALLOC_ID"], description = "ID分配") @@ -54,4 +55,17 @@ interface ServiceAllocIdResource { @BkField(minLength = 1, maxLength = 128) bizTag: String ): Result + + @GET + @Path("/types/segment/tags/{bizTag}/batchGenerate") + @ApiOperation("按号段模式批量生成Id(本质是for循环实现,减少远程调用)") + fun batchGenerateSegmentId( + @ApiParam("业务标签", required = true) + @PathParam("bizTag") + @BkField(minLength = 1, maxLength = 128) + bizTag: String, + @ApiParam("个数", required = true) + @QueryParam("number") + number: Int + ): Result> } diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceProjectResource.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceProjectResource.kt index 2a9722a790e..900f4c8cacd 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceProjectResource.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceProjectResource.kt @@ -29,6 +29,9 @@ package com.tencent.devops.project.api.service import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_ACCESS_TOKEN import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_USER_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE +import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.project.pojo.OrgInfo import com.tencent.devops.project.pojo.ProjectBaseInfo import com.tencent.devops.project.pojo.ProjectCreateInfo @@ -276,4 +279,19 @@ interface ServiceProjectResource { @ApiParam("添加信息", required = true) createInfo: ProjectCreateUserInfo ): Result + + @ApiOperation("是否拥有某实例的某action的权限") + @Path("/{projectId}/hasPermission/{permission}") + @GET + fun hasPermission( + @ApiParam("用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("项目ID", required = true) + @PathParam("projectId") + projectId: String, + @ApiParam("权限action", required = true) + @PathParam("permission") + permission: AuthPermission + ): Result } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceAllocIdResourceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceAllocIdResourceImpl.kt index 246d93e20c6..0d7e306ac7e 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceAllocIdResourceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceAllocIdResourceImpl.kt @@ -5,6 +5,7 @@ import com.tencent.devops.leaf.common.Status import com.tencent.devops.leaf.service.SegmentService import com.tencent.devops.project.api.service.ServiceAllocIdResource import com.tencent.devops.project.pojo.Result +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @RestResource @@ -19,4 +20,22 @@ class ServiceAllocIdResourceImpl @Autowired constructor( } return Result(result.id) } + + override fun batchGenerateSegmentId(bizTag: String, number: Int): Result> { + val list = mutableListOf() + for (i in 1..number) { + val result = segmentService.getId(bizTag) + if (result.status != Status.SUCCESS) { + list.add(null) + logger.warn("generate segment id failed , i:$i , bizTag:$bizTag , number:$number") + } else { + list.add(result.id) + } + } + return Result(list) + } + + companion object { + private val logger = LoggerFactory.getLogger(ServiceAllocIdResourceImpl::class.java) + } } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceProjectResourceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceProjectResourceImpl.kt index a52536e122e..7cdf36ae581 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceProjectResourceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceProjectResourceImpl.kt @@ -27,6 +27,7 @@ package com.tencent.devops.project.resources +import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.common.web.RestResource import com.tencent.devops.project.api.service.ServiceProjectResource import com.tencent.devops.project.pojo.OrgInfo @@ -180,4 +181,13 @@ class ServiceProjectResourceImpl @Autowired constructor( override fun createProjectUser(projectId: String, createInfo: ProjectCreateUserInfo): Result { return Result(projectService.createProjectUser(projectId, createInfo)) } + + override fun hasPermission(userId: String, projectId: String, permission: AuthPermission): Result { + return Result(projectService.verifyUserProjectPermission( + accessToken = null, + userId = userId, + projectId = projectId, + permission = permission + )) + } } diff --git a/src/backend/ci/core/quality/api-quality/src/main/kotlin/com/tencent/devops/quality/api/v2/pojo/request/RuleUpdateRequest.kt b/src/backend/ci/core/quality/api-quality/src/main/kotlin/com/tencent/devops/quality/api/v2/pojo/request/RuleUpdateRequest.kt index aef1421665e..71d22def5bb 100644 --- a/src/backend/ci/core/quality/api-quality/src/main/kotlin/com/tencent/devops/quality/api/v2/pojo/request/RuleUpdateRequest.kt +++ b/src/backend/ci/core/quality/api-quality/src/main/kotlin/com/tencent/devops/quality/api/v2/pojo/request/RuleUpdateRequest.kt @@ -60,7 +60,7 @@ data class RuleUpdateRequest( val auditUserList: List?, @ApiModelProperty("审核超时时间", required = false) val auditTimeoutMinutes: Int?, - @ApiModelProperty("红线匹配的id", required = false) + @ApiModelProperty("红线匹配的id(必填)", required = true) val gatewayId: String? ) { data class CreateRequestIndicator( diff --git a/src/backend/ci/core/quality/api-quality/src/main/kotlin/com/tencent/devops/quality/pojo/RuleInterceptHistory.kt b/src/backend/ci/core/quality/api-quality/src/main/kotlin/com/tencent/devops/quality/pojo/RuleInterceptHistory.kt index a85aee842e3..2941a665628 100644 --- a/src/backend/ci/core/quality/api-quality/src/main/kotlin/com/tencent/devops/quality/pojo/RuleInterceptHistory.kt +++ b/src/backend/ci/core/quality/api-quality/src/main/kotlin/com/tencent/devops/quality/pojo/RuleInterceptHistory.kt @@ -34,7 +34,7 @@ import io.swagger.annotations.ApiModelProperty @ApiModel("质量红线-拦截记录") data class RuleInterceptHistory( - @ApiModelProperty("hashId", required = true) + @ApiModelProperty("hashId 红线拦截记录在表中主键Id的哈希值，是唯一的", required = true) val hashId: String, @ApiModelProperty("项目里的序号", required = true) val num: Long, diff --git a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/cron/QualityBuildHisJob.kt b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/cron/QualityBuildHisJob.kt index d035172e157..95758cf073c 100644 --- a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/cron/QualityBuildHisJob.kt +++ b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/cron/QualityBuildHisJob.kt @@ -45,11 +45,18 @@ class QualityBuildHisJob @Autowired constructor( private val logger = LoggerFactory.getLogger(QualityBuildHisJob::class.java) - @Value("\${quality.build.his.clean.timeGap:15}") + @Value("\${quality.buildHis.clean.timeGap:15}") var cleanTimeGapDay: Long = 12 + @Value("\${quality.buildHis.clean.enable:#{false}}") + val cleanEnable: Boolean = false + @Scheduled(cron = "0 0 6 * * ?") fun clean() { + if (!cleanEnable) { + logger.info("quality buildHis daily clean disabled.") + return + } val key = this::class.java.name + "#" + Thread.currentThread().stackTrace[1].methodName val lock = RedisLock(redisOperation, key, 3600L) try { diff --git a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/cron/QualityHisMetadataJob.kt b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/cron/QualityHisMetadataJob.kt index 51293b023f9..eb965395ff4 100644 --- a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/cron/QualityHisMetadataJob.kt +++ b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/cron/QualityHisMetadataJob.kt @@ -48,11 +48,11 @@ class QualityHisMetadataJob @Autowired constructor( private val logger = LoggerFactory.getLogger(QualityHisMetadataJob::class.java) - @Value("\${quality.metadata.clean.timeGap:12}") - var cleanTimeGapHour: Long = 12 + @Value("\${quality.metadata.clean.timeGap:31}") + var cleanTimeGapDay: Long = 31 - @Value("\${quality.metadata.clean.round:100}") - var cleanRound: Long = 100 + @Value("\${quality.metadata.clean.round:400}") + var cleanRound: Long = 400 @Value("\${quality.metadata.clean.roundSize:10000}") var roundSize: Long = 10000 @@ -60,8 +60,15 @@ class QualityHisMetadataJob @Autowired constructor( @Value("\${quality.metadata.clean.roundGap:5}") var roundGap: Long = 5 + @Value("\${quality.metadata.clean.enable:#{false}}") + val cleanEnable: Boolean = false + @Scheduled(cron = "0 0 6 * * ?") fun clean() { + if (!cleanEnable) { + logger.info("quality metadata daily clean disabled.") + return + } val key = this::class.java.name + "#" + Thread.currentThread().stackTrace[1].methodName val lock = RedisLock(redisOperation, key, 3600L) try { @@ -70,16 +77,16 @@ class QualityHisMetadataJob @Autowired constructor( return } - val deleteTime = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(cleanTimeGapHour) + val deleteTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(cleanTimeGapDay) logger.info("start to delete quality his meta data: " + - "$deleteTime, $cleanTimeGapHour, $cleanRound， $roundSize, $roundGap") + "$deleteTime, $cleanTimeGapDay, $cleanRound, $roundSize, $roundGap") // 执行cleanRound轮清理详情数据操作 for (i in 1..cleanRound) { // 分页读取12个小时前的数据和创建时间为null的历史数据 val result = - qualityHisMetadataDao.getHisMetadataByCreateTime(dslContext, deleteTime, (i - 1) * roundSize) + qualityHisMetadataDao.getHisMetadataByCreateTime(dslContext, deleteTime, roundSize.toInt()) // 分成两批，nullResultIds是待更新的ID，resultIds是待删除的ID val nullResultIds = mutableSetOf() diff --git a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/dao/v2/QualityHisMetadataDao.kt b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/dao/v2/QualityHisMetadataDao.kt index 614322ffe5d..296f1f31ac5 100644 --- a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/dao/v2/QualityHisMetadataDao.kt +++ b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/dao/v2/QualityHisMetadataDao.kt @@ -127,13 +127,11 @@ class QualityHisMetadataDao { fun getHisMetadataByCreateTime( dslContext: DSLContext, time: Long, - offset: Long, pageSize: Int = 10000 ): Result { return with(TQualityHisDetailMetadata.T_QUALITY_HIS_DETAIL_METADATA) { dslContext.selectFrom(this) .where(CREATE_TIME.lt(time).or(CREATE_TIME.isNull)) - .offset(offset) .limit(pageSize) .fetch() } diff --git a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ExternalCodeccRepoResource.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ExternalCodeccRepoResource.kt index a324bf8a6ac..f052d888153 100644 --- a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ExternalCodeccRepoResource.kt +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ExternalCodeccRepoResource.kt @@ -164,6 +164,18 @@ interface ExternalCodeccRepoResource { userId: String ): Result> + @ApiOperation("获取代码库有权限成员列表") + @GET + @Path("/isProjectMember") + fun isProjectMember( + @ApiParam(value = "代码库url") + @QueryParam("repoName") + repoUrl: String, + @ApiParam(value = "用户id") + @QueryParam("userId") + userId: String + ): Result + @ApiOperation("通过凭证Id获取文件内容") @GET @Path("/{projectId}/getFileContentByUrl") diff --git a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceGitResource.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceGitResource.kt index cf572b36617..6811219a0f2 100644 --- a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceGitResource.kt +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceGitResource.kt @@ -745,6 +745,21 @@ interface ServiceGitResource { tokenType: TokenTypeEnum ): Result> + @ApiOperation("开启git仓库ci") + @GET + @Path("/stream/gitEnableCi") + fun enableCi( + @ApiParam(value = "仓库id或编码过的仓库path") + @QueryParam("projectName") + projectName: String, + @QueryParam("token") + token: String, + @QueryParam("tokenType") + tokenType: TokenTypeEnum, + @QueryParam("enable") + enable: Boolean? = true + ): Result + @ApiOperation("工蜂创建文件") @POST @Path("/gitcode/create/file") diff --git a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/commit/CommitResponse.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/commit/CommitResponse.kt index 942243141b9..872c2441676 100644 --- a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/commit/CommitResponse.kt +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/commit/CommitResponse.kt @@ -32,7 +32,7 @@ import io.swagger.annotations.ApiModelProperty @ApiModel("提交返回模型") data class CommitResponse( - @ApiModelProperty("名称") + @ApiModelProperty("仓库名称") val name: String, @ApiModelProperty("插件ID") val elementId: String, diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ExternalCodeccRepoResourceImpl.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ExternalCodeccRepoResourceImpl.kt index 37ee099c7d1..b5bd78d1d2d 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ExternalCodeccRepoResourceImpl.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ExternalCodeccRepoResourceImpl.kt @@ -126,6 +126,10 @@ class ExternalCodeccRepoResourceImpl @Autowired constructor( return commonRepoFileService.getGitProjectAllMembers(repoUrl, userId) } + override fun isProjectMember(repoUrl: String, userId: String): Result { + return commonRepoFileService.isProjectMember(repoUrl = repoUrl, userId = userId) + } + override fun getFileContentByUrl( projectId: String, repoUrl: String, diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceGitResourceImpl.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceGitResourceImpl.kt index 2399e28aa39..1e695271de6 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceGitResourceImpl.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceGitResourceImpl.kt @@ -541,6 +541,17 @@ class ServiceGitResourceImpl @Autowired constructor( ) } + override fun enableCi( + projectName: String, + token: String, + tokenType: TokenTypeEnum, + enable: Boolean? + ): Result { + return gitService.enableCi( + projectName = projectName, token = token, tokenType = tokenType, enable = enable + ) + } + override fun gitCreateFile( gitProjectId: String, token: String, diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/CommonRepoFileService.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/CommonRepoFileService.kt index 167fed31a3c..be6e571bc27 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/CommonRepoFileService.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/CommonRepoFileService.kt @@ -33,6 +33,7 @@ import com.tencent.devops.common.api.util.AESUtil import com.tencent.devops.common.service.utils.MessageCodeUtil import com.tencent.devops.repository.dao.GitTokenDao import com.tencent.devops.repository.pojo.enums.RepoAuthType +import com.tencent.devops.repository.pojo.enums.TokenTypeEnum import com.tencent.devops.repository.service.scm.IGitService import com.tencent.devops.scm.pojo.GitMember import com.tencent.devops.scm.utils.code.git.GitUtils @@ -118,4 +119,27 @@ class CommonRepoFileService @Autowired constructor( ) ) } + + fun isProjectMember(repoUrl: String, userId: String): Result { + val token = AESUtil.decrypt( + key = aesKey, + content = gitTokenDao.getAccessToken(dslContext, userId)?.accessToken + ?: return MessageCodeUtil.generateResponseDataObject(CommonMessageCode.OAUTH_TOKEN_IS_INVALID) + ) + val projectUser = gitService.getProjectMembersAll( + gitProjectId = GitUtils.getProjectName(repoUrl), + page = 1, + pageSize = 10, + search = userId, + tokenType = TokenTypeEnum.OAUTH, + token = token + ).data?.map { + it.username + } ?: emptyList() + return if (projectUser.isNotEmpty() && projectUser.contains(userId)) { + Result(true) + } else { + Result(false) + } + } } diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/GitService.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/GitService.kt index bcac0c9af2d..4cbeff41191 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/GitService.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/GitService.kt @@ -1110,10 +1110,10 @@ class GitService @Autowired constructor( ): GitMrChangeInfo { val url = StringBuilder( "${getApiUrl(repoUrl)}/projects/${ - URLEncoder.encode( - id, - "UTF-8" - ) + URLEncoder.encode( + id, + "UTF-8" + ) }/merge_request/$mrId/changes" ) logger.info("get mr changes info url: $url") @@ -1750,6 +1750,45 @@ class GitService @Autowired constructor( } } + @BkTimed(extraTags = ["operation", "enableCi"], value = "bk_tgit_api_time") + override fun enableCi( + projectName: String, + token: String, + tokenType: TokenTypeEnum, + enable: Boolean? + ): Result { + logger.info( + "enableCi projectName:$projectName," + + "enable:$enable,tokenType:$tokenType" + ) + val encodeProjectName = URLEncoder.encode(projectName, "utf-8") + val url = StringBuilder("${gitConfig.gitApiUrl}/projects/$encodeProjectName/ci/enable") + setToken(tokenType, url, token) + url.append("&enable_ci=$enable") + val request = Request.Builder() + .url(url.toString()) + .put( + RequestBody.create( + MediaType.parse("application/json;charset=utf-8"), "{}" + ) + ) + .build() + OkhttpUtils.doHttp(request).use { + if (!it.isSuccessful) { + return Result(it.code(), "enableCi fail ${it.message()}") + } + val data = it.body()!!.string() + logger.info("enableCi response>> $data") + val dataMap = JsonUtil.toMap(data) + val code = dataMap["code"] + if (code != 200) { + // 把工蜂的错误提示抛出去 + return Result(code as Int, "${dataMap["message"]}") + } + return Result(true) + } + } + @BkTimed(extraTags = ["operation", "git_create_file"], value = "bk_tgit_api_time") override fun gitCreateFile( gitProjectId: String, @@ -1777,7 +1816,16 @@ class GitService @Autowired constructor( } } - override fun getGitCodeProjectList(accessToken: String, page: Int?, pageSize: Int?, search: String?, orderBy: GitCodeProjectsOrder?, sort: GitCodeBranchesSort?, owned: Boolean?, minAccessLevel: GitAccessLevelEnum?): Result> { + override fun getGitCodeProjectList( + accessToken: String, + page: Int?, + pageSize: Int?, + search: String?, + orderBy: GitCodeProjectsOrder?, + sort: GitCodeBranchesSort?, + owned: Boolean?, + minAccessLevel: GitAccessLevelEnum? + ): Result> { val pageNotNull = page ?: 1 val pageSizeNotNull = pageSize ?: 20 val url = "$gitCIUrl/api/v3/projects?access_token=$accessToken&page=$pageNotNull&per_page=$pageSizeNotNull" diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/IGitService.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/IGitService.kt index 2a820b13b55..2178ada11e8 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/IGitService.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/IGitService.kt @@ -319,6 +319,13 @@ interface IGitService { tokenType: TokenTypeEnum ): Result> + fun enableCi( + projectName: String, + token: String, + tokenType: TokenTypeEnum, + enable: Boolean ? = true + ): Result + fun gitCreateFile( gitProjectId: String, token: String, diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/atom/OpAtomResource.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/atom/OpAtomResource.kt index 7773ef1033a..7cc0103e8aa 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/atom/OpAtomResource.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/atom/OpAtomResource.kt @@ -42,6 +42,9 @@ import com.tencent.devops.store.pojo.atom.enums.OpSortTypeEnum import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam +import org.glassfish.jersey.media.multipart.FormDataContentDisposition +import org.glassfish.jersey.media.multipart.FormDataParam +import java.io.InputStream import javax.ws.rs.Consumes import javax.ws.rs.DELETE import javax.ws.rs.GET @@ -187,4 +190,22 @@ interface OpAtomResource { @ApiParam("下架插件请求报文") atomOfflineReq: AtomOfflineReq ): Result + + @ApiOperation("根据插件包一键部署插件") + @POST + @Path("/deploy") + @Consumes(MediaType.MULTIPART_FORM_DATA) + fun releaseAtom( + @ApiParam(value = "用户ID", required = true, defaultValue = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("atomCode", required = true) + @FormDataParam("atomCode") + atomCode: String, + @ApiParam("文件", required = true) + @FormDataParam("file") + inputStream: InputStream, + @FormDataParam("file") + disposition: FormDataContentDisposition + ): Result } diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/atom/UserAtomReleaseResource.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/atom/UserAtomReleaseResource.kt index 45aa5388f77..f5f31c2ae42 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/atom/UserAtomReleaseResource.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/atom/UserAtomReleaseResource.kt @@ -64,7 +64,7 @@ interface UserAtomReleaseResource { @ApiParam("插件市场工作台-新增插件请求报文体", required = true) @Valid marketAtomCreateRequest: MarketAtomCreateRequest - ): Result + ): Result @ApiOperation("插件工作台-升级插件") @PUT diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/ServicePublishersResource.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/BuildPublishersResource.kt similarity index 95% rename from src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/ServicePublishersResource.kt rename to src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/BuildPublishersResource.kt index 564540433bb..8f443d228a7 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/ServicePublishersResource.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/BuildPublishersResource.kt @@ -43,15 +43,15 @@ import javax.ws.rs.Produces import javax.ws.rs.QueryParam import javax.ws.rs.core.MediaType -@Api(tags = ["SERVICE_PUBLISHER"], description = "service-publisher") -@Path("/service/publisher/sync") +@Api(tags = ["BUILD_PUBLISHER"], description = "build_publisher") +@Path("/build/store/publisher/sync") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) -interface ServicePublishersResource { +interface BuildPublishersResource { @ApiOperation("同步新增发布者信息") @POST - @Path("/publisher/add") + @Path("/add") fun synAddPublisherData( @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) @ApiParam("用户ID", required = true) @@ -62,7 +62,7 @@ interface ServicePublishersResource { @ApiOperation("同步删除发布者信息") @DELETE - @Path("/publisher/delete") + @Path("/delete") fun synDeletePublisherData( @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) @ApiParam("用户ID", required = true) @@ -73,7 +73,7 @@ interface ServicePublishersResource { @ApiOperation("同步更新发布者信息") @POST - @Path("/publisher/update") + @Path("/update") fun synUpdatePublisherData( @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) @ApiParam("用户ID", required = true) diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/ServiceStoreLogoResource.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/ServiceStoreLogoResource.kt new file mode 100644 index 00000000000..fb18073bff4 --- /dev/null +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/ServiceStoreLogoResource.kt @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.api.common + +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.store.pojo.common.StoreLogoInfo +import io.swagger.annotations.Api +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import org.glassfish.jersey.media.multipart.FormDataContentDisposition +import org.glassfish.jersey.media.multipart.FormDataParam +import java.io.InputStream +import javax.ws.rs.Consumes +import javax.ws.rs.HeaderParam +import javax.ws.rs.POST +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.core.MediaType + +@Api(tags = ["SERVICE_STORE_LOGO"], description = "STORE-LOGO") +@Path("/service/store/logo") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface ServiceStoreLogoResource { + + @ApiOperation("上传logo") + @POST + @Path("/upload") + @Consumes(MediaType.MULTIPART_FORM_DATA) + fun uploadStoreLogo( + @ApiParam("userId", required = true) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @ApiParam("contentLength", required = true) + @HeaderParam("content-length") + contentLength: Long, + @ApiParam("logo", required = true) + @FormDataParam("logo") + inputStream: InputStream, + @FormDataParam("logo") + disposition: FormDataContentDisposition + ): Result +} diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/constant/StoreMessageCode.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/constant/StoreMessageCode.kt index 15d1bf70813..044a64241c0 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/constant/StoreMessageCode.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/constant/StoreMessageCode.kt @@ -145,4 +145,5 @@ object StoreMessageCode { const val SENSITIVE_API_APPROVED_IS_NOT_ALLOW_PASS = "2120911" // 研发商店：敏感API已经取消不能审批 const val SENSITIVE_API_NOT_EXIST = "2120912" // 研发商店：敏感API[{0}]不存在 const val USER_HIS_VERSION_UPGRADE_INVALID = "2120913" // 研发商店：当前发布类型下仅能新增历史大版本下的小版本，请修改版本号或者发布类型 + const val USER_UPLOAD_FILE_PATH_ERROR = "2120914" // 研发商店：文件路径[{0}]错误 } diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/atom/AtomConfigInfo.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/atom/AtomConfigInfo.kt new file mode 100644 index 00000000000..1c9d19e625c --- /dev/null +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/atom/AtomConfigInfo.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.pojo.atom + +import com.tencent.devops.common.api.enums.FrontendTypeEnum +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("插件配置信息") +data class AtomConfigInfo( + @ApiModelProperty(value = "前端UI渲染方式", required = true) + val frontendType: FrontendTypeEnum = FrontendTypeEnum.NORMAL +) diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/atom/ReleaseInfo.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/atom/ReleaseInfo.kt new file mode 100644 index 00000000000..01a03a88841 --- /dev/null +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/atom/ReleaseInfo.kt @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.pojo.atom + +import com.fasterxml.jackson.annotation.JsonProperty +import com.tencent.devops.common.web.annotation.BkField +import com.tencent.devops.common.web.constant.BkStyleEnum +import com.tencent.devops.store.pojo.atom.enums.AtomCategoryEnum +import com.tencent.devops.store.pojo.atom.enums.JobTypeEnum +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("插件发布部署模型") +data class ReleaseInfo( + @ApiModelProperty("项目编码", required = true) + var projectId: String, + @ApiModelProperty("插件名称", required = true) + @field:BkField(patternStyle = BkStyleEnum.NAME_STYLE) + var name: String, + @ApiModelProperty("开发语言", required = true) + @field:BkField(patternStyle = BkStyleEnum.LANGUAGE_STYLE) + var language: String, + @ApiModelProperty("插件logo地址", required = true) + @field:BkField(maxLength = 1024) + var logoUrl: String, + @ApiModelProperty("支持的操作系统", required = true) + val os: ArrayList, + @ApiModelProperty(value = "插件配置信息", required = true) + val configInfo: AtomConfigInfo, + @ApiModelProperty("插件所属范畴", required = true) + val category: AtomCategoryEnum, + @ApiModelProperty("所属插件分类代码", required = true) + val classifyCode: String, + @ApiModelProperty("适用Job类型", required = true) + val jobType: JobTypeEnum, + @JsonProperty(value = "labelCodes", required = false) + @ApiModelProperty("标签id集合", name = "labelCodes") + val labelCodes: ArrayList? = null, + @ApiModelProperty("版本信息", required = true) + val versionInfo: VersionInfo, + @ApiModelProperty("插件简介", required = true) + @field:BkField(maxLength = 256) + val summary: String, + @ApiModelProperty("插件描述", required = true) + @field:BkField(maxLength = 65535) + var description: String +) diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/atom/VersionInfo.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/atom/VersionInfo.kt new file mode 100644 index 00000000000..d770114b52a --- /dev/null +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/atom/VersionInfo.kt @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.pojo.atom + +import com.tencent.devops.common.web.annotation.BkField +import com.tencent.devops.store.pojo.common.enums.ReleaseTypeEnum +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("版本信息") +data class VersionInfo( + @ApiModelProperty("发布者", required = true) + val publisher: String, + @ApiModelProperty("发布类型", required = true) + val releaseType: ReleaseTypeEnum, + @ApiModelProperty("插件版本", required = true) + val version: String, + @ApiModelProperty("版本日志内容", required = true) + @field:BkField(maxLength = 65535) + val versionContent: String +) diff --git a/src/backend/ci/core/store/biz-store-image/src/main/kotlin/com/tencent/devops/store/service/image/ImageBuildResourceServiceImpl.kt b/src/backend/ci/core/store/biz-store-image/src/main/kotlin/com/tencent/devops/store/service/image/ImageBuildResourceServiceImpl.kt index fc09f5ed160..860fbfb3554 100644 --- a/src/backend/ci/core/store/biz-store-image/src/main/kotlin/com/tencent/devops/store/service/image/ImageBuildResourceServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store-image/src/main/kotlin/com/tencent/devops/store/service/image/ImageBuildResourceServiceImpl.kt @@ -60,7 +60,6 @@ class ImageBuildResourceServiceImpl @Autowired constructor( override fun getDefaultBuildResource(buildType: BuildType): Any? { logger.info("getDefaultBuildResource buildType=${buildType.name}") if (buildType.name == BuildType.DOCKER.name || - buildType.name == BuildType.IDC.name || buildType.name == BuildType.PUBLIC_DEVCLOUD.name || buildType.name == BuildType.KUBERNETES.name || buildType.name == BuildType.PUBLIC_BCS.name) { diff --git a/src/backend/ci/core/store/biz-store-sample/src/main/kotlin/com/tencent/devops/store/service/atom/impl/SampleAtomReleaseServiceImpl.kt b/src/backend/ci/core/store/biz-store-sample/src/main/kotlin/com/tencent/devops/store/service/atom/impl/SampleAtomReleaseServiceImpl.kt index 2d7f026a28a..8a05334ce5f 100644 --- a/src/backend/ci/core/store/biz-store-sample/src/main/kotlin/com/tencent/devops/store/service/atom/impl/SampleAtomReleaseServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store-sample/src/main/kotlin/com/tencent/devops/store/service/atom/impl/SampleAtomReleaseServiceImpl.kt @@ -41,8 +41,8 @@ import com.tencent.devops.common.api.constant.TEST import com.tencent.devops.common.api.constant.UNDO import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.service.utils.MessageCodeUtil -import com.tencent.devops.store.constant.StoreMessageCode import com.tencent.devops.model.store.tables.records.TAtomRecord +import com.tencent.devops.store.constant.StoreMessageCode import com.tencent.devops.store.pojo.atom.AtomReleaseRequest import com.tencent.devops.store.pojo.atom.MarketAtomCreateRequest import com.tencent.devops.store.pojo.atom.MarketAtomUpdateRequest diff --git a/src/backend/ci/core/store/biz-store-sample/src/test/kotlin/com/tencent/devops/store/util/AtomReleaseTxtAnalysisUtilTest.kt b/src/backend/ci/core/store/biz-store-sample/src/test/kotlin/com/tencent/devops/store/util/AtomReleaseTxtAnalysisUtilTest.kt new file mode 100644 index 00000000000..187f89e493a --- /dev/null +++ b/src/backend/ci/core/store/biz-store-sample/src/test/kotlin/com/tencent/devops/store/util/AtomReleaseTxtAnalysisUtilTest.kt @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.util + +import com.tencent.devops.store.utils.AtomReleaseTxtAnalysisUtil +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class AtomReleaseTxtAnalysisUtilTest { + + @Test + fun regexAnalysisTest() { + val input = "插件发布测试描述:\${{indexFile(\"cat2.png\")}}||插件发布测试描述:\${{indexFile(\"cat.png\")}}" + val pathList = mutableListOf() + val result = mutableMapOf() + AtomReleaseTxtAnalysisUtil.regexAnalysis( + input = input, + atomPath = "", + pathList = pathList + ) + pathList.forEach { + result[it] = "www.tested.xxx" + } + val filePathReplaceResult = AtomReleaseTxtAnalysisUtil.filePathReplace(result, input) + Assertions.assertEquals( + "插件发布测试描述:![cat2.png](www.tested.xxx)||插件发布测试描述:![cat.png](www.tested.xxx)", + filePathReplaceResult + ) + } +} diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/dao/common/LabelDao.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/dao/common/LabelDao.kt index 475d872c94e..95a17a59c92 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/dao/common/LabelDao.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/dao/common/LabelDao.kt @@ -75,6 +75,13 @@ class LabelDao { } } + fun getIdsByCodes(dslContext: DSLContext, labelCodes: List, type: Byte): List { + with(TLabel.T_LABEL) { + return dslContext.select(ID).from(this).where(LABEL_CODE.`in`(labelCodes).and(TYPE.eq(type))) + .fetchInto(String::class.java) + } + } + fun delete(dslContext: DSLContext, id: String) { with(TLabel.T_LABEL) { dslContext.deleteFrom(this) diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/atom/OpAtomResourceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/atom/OpAtomResourceImpl.kt index 9375e99ffea..671c2783201 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/atom/OpAtomResourceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/atom/OpAtomResourceImpl.kt @@ -43,7 +43,9 @@ import com.tencent.devops.store.service.atom.AtomReleaseService import com.tencent.devops.store.service.atom.AtomService import com.tencent.devops.store.service.atom.MarketAtomService import com.tencent.devops.store.service.atom.OpAtomService +import org.glassfish.jersey.media.multipart.FormDataContentDisposition import org.springframework.beans.factory.annotation.Autowired +import java.io.InputStream @RestResource class OpAtomResourceImpl @Autowired constructor( @@ -118,4 +120,18 @@ class OpAtomResourceImpl @Autowired constructor( checkPermissionFlag = false ) } + + override fun releaseAtom( + userId: String, + atomCode: String, + inputStream: InputStream, + disposition: FormDataContentDisposition + ): Result { + return opAtomService.releaseAtom( + userId = userId, + atomCode = atomCode, + inputStream = inputStream, + disposition = disposition + ) + } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/atom/UserAtomReleaseResourceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/atom/UserAtomReleaseResourceImpl.kt index 621009fe483..6aafc5142a6 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/atom/UserAtomReleaseResourceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/atom/UserAtomReleaseResourceImpl.kt @@ -50,7 +50,7 @@ class UserAtomReleaseResourceImpl @Autowired constructor( return atomReleaseService.updateMarketAtom(userId, projectCode, marketAtomUpdateRequest) } - override fun addMarketAtom(userId: String, marketAtomCreateRequest: MarketAtomCreateRequest): Result { + override fun addMarketAtom(userId: String, marketAtomCreateRequest: MarketAtomCreateRequest): Result { return atomReleaseService.addMarketAtom(userId, marketAtomCreateRequest) } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/common/ServicePublishersResourceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/common/BuildPublishersResourceImpl.kt similarity index 86% rename from src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/common/ServicePublishersResourceImpl.kt rename to src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/common/BuildPublishersResourceImpl.kt index 9143d0128d2..efbb0f00408 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/common/ServicePublishersResourceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/common/BuildPublishersResourceImpl.kt @@ -28,29 +28,35 @@ package com.tencent.devops.store.resources.common import com.tencent.devops.common.web.RestResource -import com.tencent.devops.store.api.common.ServicePublishersResource +import com.tencent.devops.store.api.common.BuildPublishersResource import com.tencent.devops.store.pojo.common.PublishersRequest import com.tencent.devops.store.pojo.common.StoreDockingPlatformRequest import com.tencent.devops.store.service.common.PublishersDataService import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.annotation.SensitiveApiPermission import org.springframework.beans.factory.annotation.Autowired @RestResource -class ServicePublishersResourceImpl @Autowired constructor( +class BuildPublishersResourceImpl @Autowired constructor( private val publishersDataService: PublishersDataService -) : ServicePublishersResource { +) : BuildPublishersResource { + + @SensitiveApiPermission("syn_publisher_data") override fun synAddPublisherData(userId: String, publishers: List): Result { return Result(publishersDataService.createPublisherData(userId, publishers)) } + @SensitiveApiPermission("syn_publisher_data") override fun synDeletePublisherData(userId: String, publishers: List): Result { return Result(publishersDataService.deletePublisherData(userId, publishers)) } + @SensitiveApiPermission("syn_publisher_data") override fun synUpdatePublisherData(userId: String, publishers: List): Result { return Result(publishersDataService.updatePublisherData(userId, publishers)) } + @SensitiveApiPermission("syn_platforms_data") override fun synAddPlatformsData( userId: String, storeDockingPlatformRequests: List @@ -58,6 +64,7 @@ class ServicePublishersResourceImpl @Autowired constructor( return Result(publishersDataService.createPlatformsData(userId, storeDockingPlatformRequests)) } + @SensitiveApiPermission("syn_platforms_data") override fun synDeletePlatformsData( userId: String, storeDockingPlatformRequests: List @@ -65,6 +72,7 @@ class ServicePublishersResourceImpl @Autowired constructor( return Result(publishersDataService.deletePlatformsData(userId, storeDockingPlatformRequests)) } + @SensitiveApiPermission("syn_platforms_data") override fun synUpdatePlatformsData( userId: String, storeDockingPlatformRequests: List @@ -72,6 +80,7 @@ class ServicePublishersResourceImpl @Autowired constructor( return Result(publishersDataService.updatePlatformsData(userId, storeDockingPlatformRequests)) } + @SensitiveApiPermission("syn_platforms_data") override fun synUpdatePlatformsLogoInfo(userId: String, platformCode: String, logoUrl: String): Result { return Result(publishersDataService.updatePlatformsLogoInfo(userId, platformCode, logoUrl)) } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/common/ServiceStoreLogoResourceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/common/ServiceStoreLogoResourceImpl.kt new file mode 100644 index 00000000000..c04b82385de --- /dev/null +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/resources/common/ServiceStoreLogoResourceImpl.kt @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.resources.common + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.store.api.common.ServiceStoreLogoResource +import com.tencent.devops.store.pojo.common.StoreLogoInfo +import com.tencent.devops.store.service.common.StoreLogoService +import org.glassfish.jersey.media.multipart.FormDataContentDisposition +import org.springframework.beans.factory.annotation.Autowired +import java.io.InputStream + +@RestResource +class ServiceStoreLogoResourceImpl @Autowired constructor( + private val storeLogoService: StoreLogoService +) : ServiceStoreLogoResource { + + override fun uploadStoreLogo( + userId: String, + contentLength: Long, + inputStream: InputStream, + disposition: FormDataContentDisposition + ): Result { + return storeLogoService.uploadStoreLogo( + userId = userId, + contentLength = contentLength, + inputStream = inputStream, + disposition = disposition + ) + } +} diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/AtomReleaseService.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/AtomReleaseService.kt index fc4f06eb471..564646a4d41 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/AtomReleaseService.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/AtomReleaseService.kt @@ -30,9 +30,9 @@ package com.tencent.devops.store.service.atom import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.store.pojo.atom.AtomOfflineReq import com.tencent.devops.store.pojo.atom.AtomReleaseRequest -import com.tencent.devops.store.pojo.common.StoreProcessInfo import com.tencent.devops.store.pojo.atom.MarketAtomCreateRequest import com.tencent.devops.store.pojo.atom.MarketAtomUpdateRequest +import com.tencent.devops.store.pojo.common.StoreProcessInfo @Suppress("ALL") interface AtomReleaseService { @@ -40,7 +40,7 @@ interface AtomReleaseService { /** * 添加插件 */ - fun addMarketAtom(userId: String, marketAtomCreateRequest: MarketAtomCreateRequest): Result + fun addMarketAtom(userId: String, marketAtomCreateRequest: MarketAtomCreateRequest): Result /** * 升级插件 diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/OpAtomService.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/OpAtomService.kt index 7d51e6a4645..4cdac1378e4 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/OpAtomService.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/OpAtomService.kt @@ -34,6 +34,8 @@ import com.tencent.devops.store.pojo.atom.AtomResp import com.tencent.devops.store.pojo.atom.enums.AtomStatusEnum import com.tencent.devops.store.pojo.atom.enums.AtomTypeEnum import com.tencent.devops.store.pojo.atom.enums.OpSortTypeEnum +import org.glassfish.jersey.media.multipart.FormDataContentDisposition +import java.io.InputStream interface OpAtomService { @@ -68,4 +70,14 @@ interface OpAtomService { * 审核插件 */ fun approveAtom(userId: String, atomId: String, approveReq: ApproveReq): Result + + /** + * 一键部署发布插件 + */ + fun releaseAtom( + userId: String, + atomCode: String, + inputStream: InputStream, + disposition: FormDataContentDisposition + ): Result } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/AtomReleaseServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/AtomReleaseServiceImpl.kt index c16db5774b1..4c5a54b0de4 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/AtomReleaseServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/AtomReleaseServiceImpl.kt @@ -178,7 +178,7 @@ abstract class AtomReleaseServiceImpl @Autowired constructor() : AtomReleaseServ private fun validateAddMarketAtomReq( marketAtomCreateRequest: MarketAtomCreateRequest - ): Result { + ): Result? { val atomCode = marketAtomCreateRequest.atomCode // 判断插件代码是否存在 val codeCount = atomDao.countByCode(dslContext, atomCode) @@ -186,8 +186,7 @@ abstract class AtomReleaseServiceImpl @Autowired constructor() : AtomReleaseServ // 抛出错误提示 return MessageCodeUtil.generateResponseDataObject( CommonMessageCode.PARAMETER_IS_EXIST, - arrayOf(atomCode), - false + arrayOf(atomCode) ) } val atomName = marketAtomCreateRequest.name @@ -197,23 +196,22 @@ abstract class AtomReleaseServiceImpl @Autowired constructor() : AtomReleaseServ // 抛出错误提示 return MessageCodeUtil.generateResponseDataObject( CommonMessageCode.PARAMETER_IS_EXIST, - arrayOf(atomName), - false + arrayOf(atomName) ) } - return Result(true) + return null } @BkTimed(extraTags = ["publish", "addMarketAtom"], value = "store_publish_pipeline_atom") override fun addMarketAtom( userId: String, marketAtomCreateRequest: MarketAtomCreateRequest - ): Result { + ): Result { logger.info("addMarketAtom userId is :$userId,marketAtomCreateRequest is :$marketAtomCreateRequest") val atomCode = marketAtomCreateRequest.atomCode val validateResult = validateAddMarketAtomReq(marketAtomCreateRequest) - logger.info("the validateResult is :$validateResult") - if (validateResult.isNotOk()) { + if (validateResult != null) { + logger.info("the validateResult is :$validateResult") return validateResult } val handleAtomPackageResult = handleAtomPackage(marketAtomCreateRequest, userId, atomCode) @@ -222,9 +220,9 @@ abstract class AtomReleaseServiceImpl @Autowired constructor() : AtomReleaseServ return Result(handleAtomPackageResult.status, handleAtomPackageResult.message, null) } val handleAtomPackageMap = handleAtomPackageResult.data + val id = UUIDUtil.generate() dslContext.transaction { t -> val context = DSL.using(t) - val id = UUIDUtil.generate() // 添加插件基本信息 marketAtomDao.addMarketAtom( dslContext = context, @@ -278,7 +276,7 @@ abstract class AtomReleaseServiceImpl @Autowired constructor() : AtomReleaseServ storeType = StoreTypeEnum.ATOM.type.toByte() ) } - return Result(true) + return Result(id) } abstract fun handleAtomPackage( diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/MarketAtomServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/MarketAtomServiceImpl.kt index b59d5431b4e..5c50953af46 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/MarketAtomServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/MarketAtomServiceImpl.kt @@ -1108,14 +1108,10 @@ abstract class MarketAtomServiceImpl @Autowired constructor() : MarketAtomServic val propMap = JsonUtil.toMap(atom.props) val outputDataMap = propMap[ATOM_OUTPUT] as? Map return outputDataMap?.keys?.map { outputKey -> - val outputDataObj = outputDataMap[outputKey] as Map + val outputDataObj = outputDataMap[outputKey] AtomOutput( name = outputKey, - desc = if (outputDataObj[OUTPUT_DESC] == null) { - null - } else { - outputDataObj[OUTPUT_DESC].toString() - } + desc = if (outputDataObj is Map<*, *>) outputDataObj[OUTPUT_DESC]?.toString() else null ) } ?: emptyList() } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/OpAtomServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/OpAtomServiceImpl.kt index 38a13ab5611..43720a2af47 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/OpAtomServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/atom/impl/OpAtomServiceImpl.kt @@ -27,32 +27,47 @@ package com.tencent.devops.store.service.atom.impl +import com.fasterxml.jackson.core.JsonProcessingException +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.devops.artifactory.api.ServiceArchiveAtomFileResource import com.tencent.devops.common.api.constant.CommonMessageCode import com.tencent.devops.common.api.constant.INIT_VERSION +import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.api.util.DateTimeUtil import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.PageUtil +import com.tencent.devops.common.api.util.UUIDUtil +import com.tencent.devops.common.client.Client import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.service.utils.MessageCodeUtil +import com.tencent.devops.common.service.utils.ZipUtil import com.tencent.devops.model.store.tables.records.TAtomRecord import com.tencent.devops.model.store.tables.records.TClassifyRecord import com.tencent.devops.repository.pojo.enums.VisibilityLevelEnum +import com.tencent.devops.store.constant.StoreMessageCode +import com.tencent.devops.store.constant.StoreMessageCode.USER_UPLOAD_FILE_PATH_ERROR +import com.tencent.devops.store.constant.StoreMessageCode.USER_UPLOAD_PACKAGE_INVALID import com.tencent.devops.store.dao.atom.AtomDao import com.tencent.devops.store.dao.atom.MarketAtomDao import com.tencent.devops.store.dao.atom.MarketAtomFeatureDao import com.tencent.devops.store.dao.atom.MarketAtomVersionLogDao import com.tencent.devops.store.dao.common.ClassifyDao +import com.tencent.devops.store.dao.common.LabelDao import com.tencent.devops.store.pojo.atom.ApproveReq import com.tencent.devops.store.pojo.atom.Atom import com.tencent.devops.store.pojo.atom.AtomReleaseRequest import com.tencent.devops.store.pojo.atom.AtomResp +import com.tencent.devops.store.pojo.atom.MarketAtomCreateRequest +import com.tencent.devops.store.pojo.atom.MarketAtomUpdateRequest +import com.tencent.devops.store.pojo.atom.ReleaseInfo import com.tencent.devops.store.pojo.atom.enums.AtomCategoryEnum import com.tencent.devops.store.pojo.atom.enums.AtomStatusEnum import com.tencent.devops.store.pojo.atom.enums.AtomTypeEnum import com.tencent.devops.store.pojo.atom.enums.OpSortTypeEnum import com.tencent.devops.store.pojo.common.PASS import com.tencent.devops.store.pojo.common.REJECT +import com.tencent.devops.store.pojo.common.TASK_JSON_NAME import com.tencent.devops.store.pojo.common.enums.AuditTypeEnum import com.tencent.devops.store.pojo.common.enums.ReleaseTypeEnum import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum @@ -61,16 +76,24 @@ import com.tencent.devops.store.service.atom.AtomQualityService import com.tencent.devops.store.service.atom.AtomReleaseService import com.tencent.devops.store.service.atom.OpAtomService import com.tencent.devops.store.service.atom.action.AtomDecorateFactory +import com.tencent.devops.store.service.common.StoreLogoService import com.tencent.devops.store.service.websocket.StoreWebsocketService +import com.tencent.devops.store.utils.AtomReleaseTxtAnalysisUtil import com.tencent.devops.store.utils.StoreUtils +import org.glassfish.jersey.media.multipart.FormDataContentDisposition import org.jooq.DSLContext import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service +import org.springframework.util.FileSystemUtils +import java.io.File +import java.io.InputStream +import java.nio.charset.Charset +import java.nio.file.Files import java.time.LocalDateTime @Service -@Suppress("LongParameterList", "LongMethod", "ReturnCount") +@Suppress("LongParameterList", "LongMethod", "ReturnCount", "ComplexMethod", "NestedBlockDepth") class OpAtomServiceImpl @Autowired constructor( private val dslContext: DSLContext, private val classifyDao: ClassifyDao, @@ -80,12 +103,16 @@ class OpAtomServiceImpl @Autowired constructor( private val marketAtomVersionLogDao: MarketAtomVersionLogDao, private val atomQualityService: AtomQualityService, private val atomNotifyService: AtomNotifyService, + private val labelDao: LabelDao, private val atomReleaseService: AtomReleaseService, + private val storeLogoService: StoreLogoService, private val storeWebsocketService: StoreWebsocketService, - private val redisOperation: RedisOperation + private val redisOperation: RedisOperation, + private val client: Client ) : OpAtomService { private val logger = LoggerFactory.getLogger(OpAtomServiceImpl::class.java) + private val fileSeparator: String = System.getProperty("file.separator") /** * op系统获取插件信息 @@ -308,4 +335,178 @@ class OpAtomServiceImpl @Autowired constructor( storeWebsocketService.sendWebsocketMessageByAtomCodeAndAtomId(atomCode, atomId) return Result(true) } + + override fun releaseAtom( + userId: String, + atomCode: String, + inputStream: InputStream, + disposition: FormDataContentDisposition + ): Result { + // 解压插件包到临时目录 + val fileName = disposition.fileName + val index = fileName.lastIndexOf(".") + val fileType = fileName.substring(index + 1) + val file = Files.createTempFile(UUIDUtil.generate(), ".$fileType").toFile() + file.outputStream().use { + inputStream.copyTo(it) + } + val atomPath = AtomReleaseTxtAnalysisUtil.buildAtomArchivePath(userId, atomCode) + if (!File(atomPath).exists()) { + ZipUtil.unZipFile(file, atomPath, false) + } + val taskJsonFile = File("$atomPath$fileSeparator$TASK_JSON_NAME") + if (!taskJsonFile.exists()) { + return MessageCodeUtil.generateResponseDataObject( + StoreMessageCode.USER_ATOM_CONF_INVALID, + arrayOf(TASK_JSON_NAME) + ) + } + val taskJsonMap: Map + val releaseInfo: ReleaseInfo + // 解析task.json文件 + try { + val taskJsonStr = taskJsonFile.readText(Charset.forName("UTF-8")) + taskJsonMap = JsonUtil.toMap(taskJsonStr).toMutableMap() + val releaseInfoMap = taskJsonMap["releaseInfo"] + releaseInfo = JsonUtil.mapTo(releaseInfoMap as Map, ReleaseInfo::class.java) + } catch (e: JsonProcessingException) { + return MessageCodeUtil.generateResponseDataObject( + StoreMessageCode.USER_REPOSITORY_TASK_JSON_FIELD_IS_INVALID, + arrayOf("releaseInfo") + ) + } + // 新增插件 + val addMarketAtomResult = atomReleaseService.addMarketAtom( + userId, + MarketAtomCreateRequest( + projectCode = releaseInfo.projectId, + atomCode = atomCode, + name = releaseInfo.name, + language = releaseInfo.language, + frontendType = releaseInfo.configInfo.frontendType + ) + ) + if (addMarketAtomResult.isNotOk()) { + return Result(data = false, message = addMarketAtomResult.message) + } + val atomId = addMarketAtomResult.data!! + // 远程logo资源不做处理 + if (!releaseInfo.logoUrl.startsWith("http")) { + // 解析logoUrl + val logoUrlAnalysisResult = AtomReleaseTxtAnalysisUtil.logoUrlAnalysis(releaseInfo.logoUrl) + if (logoUrlAnalysisResult.isNotOk()) { + return Result( + data = false, + status = logoUrlAnalysisResult.status, + message = logoUrlAnalysisResult.message + ) + } + val relativePath = logoUrlAnalysisResult.data + val logoFile = File("$atomPath${File.separator}file" + + "${File.separator}${relativePath?.removePrefix(File.separator)}") + if (logoFile.exists()) { + val result = storeLogoService.uploadStoreLogo( + userId = userId, + contentLength = logoFile.length(), + inputStream = logoFile.inputStream(), + disposition = FormDataContentDisposition( + "form-data; name=\"logo\"; filename=\"${logoFile.name}\"" + ) + ) + if (result.isOk()) { + result.data?.logoUrl?.let { releaseInfo.logoUrl = it } + } else { + return Result( + data = false, + status = result.status, + message = result.message + ) + } + } else { + throw ErrorCodeException( + errorCode = USER_UPLOAD_FILE_PATH_ERROR, + params = arrayOf(relativePath ?: "") + ) + } + } + // 解析description + releaseInfo.description = AtomReleaseTxtAnalysisUtil.descriptionAnalysis( + description = releaseInfo.description, + atomPath = atomPath, + client = client, + userId = userId + ) + taskJsonMap["releaseInfo"] = releaseInfo + // 将替换好的文本写入task.json文件 + val taskJson = taskJsonMap.toJsonString() + val fileOutputStream = taskJsonFile.outputStream() + fileOutputStream.use { + it.write(taskJson.toByteArray(charset("utf-8"))) + } + try { + if (file.exists()) { + val archiveAtomResult = AtomReleaseTxtAnalysisUtil.serviceArchiveAtomFile( + userId = userId, + projectCode = releaseInfo.projectId, + atomId = atomId, + atomCode = atomCode, + version = releaseInfo.versionInfo.version, + serviceUrlPrefix = client.getServiceUrl(ServiceArchiveAtomFileResource::class), + releaseType = releaseInfo.versionInfo.releaseType.name, + file = file, + os = JsonUtil.toJson(releaseInfo.os) + ) + if (archiveAtomResult.isNotOk()) { + return Result( + data = false, + status = archiveAtomResult.status, + message = archiveAtomResult.message + ) + } + } + } catch (ignored: Throwable) { + logger.warn("BKSystemErrorMonitor|archive atom file fail|$atomCode|error=${ignored.message}") + throw ErrorCodeException( + errorCode = USER_UPLOAD_PACKAGE_INVALID + ) + } finally { + file.delete() + FileSystemUtils.deleteRecursively(File(atomPath).parentFile) + } + val labelIds = if (releaseInfo.labelCodes != null) { + ArrayList(labelDao.getIdsByCodes(dslContext, releaseInfo.labelCodes!!, 0)) + } else null + + // 升级插件 + val updateMarketAtomResult = atomReleaseService.updateMarketAtom( + userId, + releaseInfo.projectId, + MarketAtomUpdateRequest( + atomCode = atomCode, + name = releaseInfo.name, + category = releaseInfo.category, + jobType = releaseInfo.jobType, + os = releaseInfo.os, + summary = releaseInfo.summary, + description = releaseInfo.description, + version = releaseInfo.versionInfo.version, + releaseType = releaseInfo.versionInfo.releaseType, + versionContent = releaseInfo.versionInfo.versionContent, + publisher = releaseInfo.versionInfo.publisher, + labelIdList = labelIds, + frontendType = releaseInfo.configInfo.frontendType, + logoUrl = releaseInfo.logoUrl, + classifyCode = releaseInfo.classifyCode + ) + ) + if (updateMarketAtomResult.isNotOk()) { + return Result( + data = false, + status = updateMarketAtomResult.status, + message = updateMarketAtomResult.message + ) + } + // 确认测试通过 + return atomReleaseService.passTest(userId, atomId) + } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/template/impl/MarketTemplateServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/template/impl/MarketTemplateServiceImpl.kt index b384ceb56f7..113d754fc67 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/template/impl/MarketTemplateServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/service/template/impl/MarketTemplateServiceImpl.kt @@ -675,7 +675,7 @@ abstract class MarketTemplateServiceImpl @Autowired constructor() : MarketTempla userId = userId, model = templateModel, projectCodeList = projectCodeList, - templateCode = templateCode + templateCode = templateDetail.templateCode ) } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/utils/AtomReleaseTxtAnalysisUtil.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/utils/AtomReleaseTxtAnalysisUtil.kt new file mode 100644 index 00000000000..ceea37ed092 --- /dev/null +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/utils/AtomReleaseTxtAnalysisUtil.kt @@ -0,0 +1,221 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.utils + +import com.tencent.devops.artifactory.api.ServiceArchiveAtomFileResource +import com.tencent.devops.artifactory.api.service.ServiceFileResource +import com.tencent.devops.artifactory.pojo.enums.FileChannelTypeEnum +import com.tencent.devops.common.api.constant.CommonMessageCode +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.api.util.OkhttpUtils +import com.tencent.devops.common.api.util.UUIDUtil +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.service.utils.CommonUtils +import com.tencent.devops.common.service.utils.MessageCodeUtil +import com.tencent.devops.store.constant.StoreMessageCode +import org.slf4j.LoggerFactory +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.io.InputStream +import java.net.URL +import java.util.regex.Matcher +import java.util.regex.Pattern + +object AtomReleaseTxtAnalysisUtil { + + private const val BK_CI_ATOM_DIR = "bk-atom" + private const val BK_CI_PATH_REGEX = "(\\\$\\{\\{indexFile\\()(\"[^\"]*\")" + private val fileSeparator: String = System.getProperty("file.separator") + private val logger = LoggerFactory.getLogger(AtomReleaseTxtAnalysisUtil::class.java) + private const val FILE_DEFAULT_SIZE = 1024 + + @Suppress("NestedBlockDepth") + fun descriptionAnalysis( + userId: String, + description: String, + atomPath: String, + client: Client + ): String { + val pathList = mutableListOf() + val result = mutableMapOf() + var descriptionText = description + if (description.startsWith("http") && description.endsWith(".md")) { + // 读取远程文件 + var inputStream: InputStream? = null + val file = File("$atomPath${fileSeparator}file${fileSeparator}description.md") + try { + inputStream = URL(description).openStream() + FileOutputStream(file).use { outputStream -> + var read: Int + val bytes = ByteArray(FILE_DEFAULT_SIZE) + while (inputStream.read(bytes).also { read = it } != -1) { + outputStream.write(bytes, 0, read) + } + } + descriptionText = file.readText() + } catch (e: IOException) { + logger.warn("get remote file fail:${e.message}") + } finally { + inputStream?.close() + file.delete() + } + } + descriptionText = regexAnalysis( + input = descriptionText, + atomPath = atomPath, + pathList = pathList + ) + val uploadFileToPathResult = uploadFileToPath( + userId = userId, + pathList = pathList, + client = client, + atomPath = atomPath, + result = result + ) + return filePathReplace(uploadFileToPathResult.toMutableMap(), descriptionText) + } + + private fun getAtomBasePath(): String { + return System.getProperty("java.io.tmpdir").removeSuffix(fileSeparator) + } + + fun regexAnalysis( + input: String, + atomPath: String, + pathList: MutableList + ): String { + var descriptionContent = input + val pattern: Pattern = Pattern.compile(BK_CI_PATH_REGEX) + val matcher: Matcher = pattern.matcher(descriptionContent) + while (matcher.find()) { + val path = matcher.group(2).replace("\"", "").removePrefix(fileSeparator) + if (path.endsWith(".md")) { + val file = File("$atomPath${fileSeparator}file$fileSeparator$path") + if (file.exists()) { + descriptionContent = regexAnalysis( + input = file.readText(), + atomPath = atomPath, + pathList = pathList + ) + } + } else { + pathList.add(path) + } + } + return descriptionContent + } + + fun filePathReplace( + result: MutableMap, + descriptionContent: String + ): String { + var content = descriptionContent + // 替换资源路径 + result.forEach { + val analysisPattern: Pattern = Pattern.compile("(\\\$\\{\\{indexFile\\(\"${it.key}\"\\)}})") + val analysisMatcher: Matcher = analysisPattern.matcher(content) + content = analysisMatcher.replaceFirst( + "![${it.key}](${it.value.replace(fileSeparator, "\\$fileSeparator")})" + ) + } + return content + } + + private fun uploadFileToPath( + userId: String, + pathList: List, + client: Client, + atomPath: String, + result: MutableMap + ): Map { + client.getServiceUrl(ServiceArchiveAtomFileResource::class) + pathList.forEach { path -> + val file = File("$atomPath${fileSeparator}file$fileSeparator$path") + if (file.exists()) { + val serviceUrlPrefix = client.getServiceUrl(ServiceFileResource::class) + val fileUrl = CommonUtils.serviceUploadFile( + userId = userId, + serviceUrlPrefix = serviceUrlPrefix, + file = file, + fileChannelType = FileChannelTypeEnum.WEB_SHOW.name, + logo = true + ).data + fileUrl?.let { result[path] = StoreUtils.removeUrlHost(fileUrl) } + } else { + logger.warn("Resource file does not exist:${file.path}") + } + file.delete() + } + return result + } + + fun logoUrlAnalysis(logoUrl: String): Result { + // 正则解析 + val pattern: Pattern = Pattern.compile(BK_CI_PATH_REGEX) + val matcher: Matcher = pattern.matcher(logoUrl) + val relativePath = if (matcher.find()) { + matcher.group(2).replace("\"", "") + } else null + return if (relativePath.isNullOrBlank()) { + MessageCodeUtil.generateResponseDataObject( + StoreMessageCode.USER_REPOSITORY_TASK_JSON_FIELD_IS_INVALID, + arrayOf("releaseInfo.logoUrl") + ) + } else { + Result(relativePath) + } + } + + fun serviceArchiveAtomFile( + userId: String, + projectCode: String, + atomId: String, + atomCode: String, + serviceUrlPrefix: String, + releaseType: String, + version: String, + file: File, + os: String + ): Result { + val serviceUrl = "$serviceUrlPrefix/service/artifactories/archiveAtom" + + "?userId=$userId&projectCode=$projectCode&atomId=$atomId&atomCode=$atomCode" + + "&version=$version&releaseType=$releaseType&os=$os" + OkhttpUtils.uploadFile(serviceUrl, file).use { response -> + response.body()!!.string() + if (!response.isSuccessful) { + return MessageCodeUtil.generateResponseDataObject(CommonMessageCode.SYSTEM_ERROR) + } + return Result(true) + } + } + + fun buildAtomArchivePath(userId: String, atomCode: String) = + "${getAtomBasePath()}$fileSeparator$BK_CI_ATOM_DIR$fileSeparator$userId$fileSeparator$atomCode" + + "$fileSeparator${UUIDUtil.generate()}" +} diff --git a/src/backend/ci/core/stream/api-stream/src/main/kotlin/com/tencent/devops/stream/api/user/UserStreamUserMessageResource.kt b/src/backend/ci/core/stream/api-stream/src/main/kotlin/com/tencent/devops/stream/api/user/UserStreamUserMessageResource.kt index fa5d605c853..c0fabc29d6d 100644 --- a/src/backend/ci/core/stream/api-stream/src/main/kotlin/com/tencent/devops/stream/api/user/UserStreamUserMessageResource.kt +++ b/src/backend/ci/core/stream/api-stream/src/main/kotlin/com/tencent/devops/stream/api/user/UserStreamUserMessageResource.kt @@ -68,6 +68,12 @@ interface UserStreamUserMessageResource { @ApiParam(value = "是否已读") @QueryParam("haveRead") haveRead: Boolean?, + @ApiParam(value = "消息唯一id") + @QueryParam("messageId") + messageId: String?, + @ApiParam(value = "触发人") + @QueryParam("triggerUserId") + triggerUserId: String?, @ApiParam(value = "页码") @QueryParam("page") page: Int?, diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/dao/GitRequestEventBuildDao.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/dao/GitRequestEventBuildDao.kt index bbc1960d0b5..ef1330653be 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/dao/GitRequestEventBuildDao.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/dao/GitRequestEventBuildDao.kt @@ -45,7 +45,7 @@ import org.springframework.stereotype.Repository import java.sql.Timestamp import java.time.LocalDateTime -@Suppress("ComplexCondition") +@Suppress("ComplexCondition", "ComplexMethod") @Repository class GitRequestEventBuildDao { fun save( @@ -501,7 +501,8 @@ class GitRequestEventBuildDao { buildStatus: Set?, limit: Int, offset: Int, - pipelineIds: Set? + pipelineIds: Set?, + buildIds: Set? ): List { with(TGitRequestEventBuild.T_GIT_REQUEST_EVENT_BUILD) { return getRequestEventBuildListMultiple( @@ -514,7 +515,8 @@ class GitRequestEventBuildDao { event = event, commitMsg = commitMsg, buildStatus = buildStatus, - pipelineIds = pipelineIds + pipelineIds = pipelineIds, + buildIds = buildIds ).orderBy(EVENT_ID.desc(), CREATE_TIME.desc()).limit(limit).offset(offset).fetch() } } @@ -529,7 +531,8 @@ class GitRequestEventBuildDao { event: Set?, commitMsg: String?, buildStatus: Set?, - pipelineIds: Set? + pipelineIds: Set?, + buildIds: Set? ): Int { with(TGitRequestEventBuild.T_GIT_REQUEST_EVENT_BUILD) { val dsl = dslContext.selectCount().from(this) @@ -569,6 +572,9 @@ class GitRequestEventBuildDao { if (!pipelineIds.isNullOrEmpty()) { dsl.and(PIPELINE_ID.`in`(pipelineIds)) } + if (!buildIds.isNullOrEmpty()) { + dsl.and(BUILD_ID.`in`(buildIds)) + } return dsl.fetchOne(0, Int::class.java)!! } } @@ -583,7 +589,8 @@ class GitRequestEventBuildDao { event: Set?, commitMsg: String?, buildStatus: Set?, - pipelineIds: Set? + pipelineIds: Set?, + buildIds: Set? ): SelectConditionStep { with(TGitRequestEventBuild.T_GIT_REQUEST_EVENT_BUILD) { val dsl = dslContext.selectFrom(this) @@ -623,6 +630,9 @@ class GitRequestEventBuildDao { if (!pipelineIds.isNullOrEmpty()) { dsl.and(PIPELINE_ID.`in`(pipelineIds)) } + if (!buildIds.isNullOrEmpty()) { + dsl.and(BUILD_ID.`in`(buildIds)) + } return dsl } } diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/dao/StreamUserMessageDao.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/dao/StreamUserMessageDao.kt index 141c8523920..27d46bea9b1 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/dao/StreamUserMessageDao.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/dao/StreamUserMessageDao.kt @@ -31,8 +31,8 @@ import com.tencent.devops.common.service.utils.CommonUtils import com.tencent.devops.model.stream.tables.TGitUserMessage import com.tencent.devops.model.stream.tables.records.TGitUserMessageRecord import com.tencent.devops.stream.pojo.message.UserMessageType +import org.jooq.Condition import org.jooq.DSLContext -import org.jooq.SelectConditionStep import org.springframework.stereotype.Repository @Repository @@ -74,26 +74,28 @@ class StreamUserMessageDao { projectId: String, messageType: UserMessageType?, haveRead: Boolean?, + messageId: String?, offset: Int, limit: Int ): List? { with(TGitUserMessage.T_GIT_USER_MESSAGE) { - val select = if (userId != null) { - selectMessage( - dslContext = dslContext, - userId = userId, - messageType = messageType, - haveRead = haveRead - ) - } else { - selectMessage( - dslContext = dslContext, - projectId = projectId, - messageType = messageType, - haveRead = haveRead - ) + val conditions = mutableListOf() + if (!userId.isNullOrBlank()) { + conditions.add(USER_ID.eq(userId)) + } + if (projectId.isNotBlank()) { + conditions.add(PROJECT_ID.eq(projectId)) } - return select.orderBy(ID.desc()) + if (messageType != null) { + conditions.add(MESSAGE_TYPE.eq(messageType.name)) + } + if (haveRead != null) { + conditions.add(HAVE_READ.eq(haveRead)) + } + if (messageId != null) { + conditions.add(MESSAGE_ID.eq(messageId)) + } + return dslContext.selectFrom(this).where(conditions).orderBy(ID.desc()) .limit(limit).offset(offset) .fetch() } @@ -104,24 +106,17 @@ class StreamUserMessageDao { userId: String?, projectId: String, messageType: UserMessageType?, + messageId: String?, haveRead: Boolean? ): Int { - val select = if (userId != null) { - selectMessageCount( - dslContext = dslContext, - userId = userId, - messageType = messageType, - haveRead = haveRead - ) - } else { - selectMessageCount( - dslContext = dslContext, - projectId = projectId, - messageType = messageType, - haveRead = haveRead - ) - } - return select + return selectMessageCount( + dslContext = dslContext, + userId = userId, + projectId = projectId, + messageType = messageType, + messageId = messageId, + haveRead = haveRead + ) } fun readMessage( @@ -174,6 +169,7 @@ class StreamUserMessageDao { dslContext = dslContext, userId = userId, messageType = null, + messageId = null, haveRead = false ) } else { @@ -181,6 +177,7 @@ class StreamUserMessageDao { dslContext = dslContext, projectId = projectId, messageType = null, + messageId = null, haveRead = false ) } @@ -200,7 +197,7 @@ class StreamUserMessageDao { dsl.and(USER_ID.eq(userId)) } return dsl.and(MESSAGE_ID.eq(messageId)) - .fetch().first() + .fetchAny() } } @@ -236,29 +233,11 @@ class StreamUserMessageDao { } } - private fun selectMessage( - dslContext: DSLContext, - projectId: String, - messageType: UserMessageType?, - haveRead: Boolean? - ): SelectConditionStep { - with(TGitUserMessage.T_GIT_USER_MESSAGE) { - val dsl = dslContext.selectFrom(this) - .where(PROJECT_ID.eq(projectId)) - if (messageType != null) { - dsl.and(MESSAGE_TYPE.eq(messageType.name)) - } - if (haveRead != null) { - dsl.and(HAVE_READ.eq(haveRead)) - } - return dsl - } - } - private fun selectMessageCount( dslContext: DSLContext, projectId: String, messageType: UserMessageType?, + messageId: String?, haveRead: Boolean? ): Int { with(TGitUserMessage.T_GIT_USER_MESSAGE) { @@ -270,53 +249,39 @@ class StreamUserMessageDao { if (haveRead != null) { dsl.and(HAVE_READ.eq(haveRead)) } - return dsl.fetchOne(0, Int::class.java)!! - } - } - - private fun selectMessage( - dslContext: DSLContext, - userId: String, - projectId: String? = null, - messageType: UserMessageType?, - haveRead: Boolean? - ): SelectConditionStep { - with(TGitUserMessage.T_GIT_USER_MESSAGE) { - val dsl = dslContext.selectFrom(this) - .where(USER_ID.eq(userId)) - if (!projectId.isNullOrBlank()) { - dsl.and(PROJECT_ID.eq(projectId)) - } - if (messageType != null) { - dsl.and(MESSAGE_TYPE.eq(messageType.name)) - } - if (haveRead != null) { - dsl.and(HAVE_READ.eq(haveRead)) + if (messageId != null) { + dsl.and(MESSAGE_ID.eq(messageId)) } - return dsl + return dsl.fetchOne(0, Int::class.java)!! } } private fun selectMessageCount( dslContext: DSLContext, - userId: String, + userId: String?, projectId: String? = null, messageType: UserMessageType?, + messageId: String?, haveRead: Boolean? ): Int { with(TGitUserMessage.T_GIT_USER_MESSAGE) { - val dsl = dslContext.selectCount().from(this) - .where(USER_ID.eq(userId)) + val conditions = mutableListOf() + if (!userId.isNullOrBlank()) { + conditions.add(USER_ID.eq(userId)) + } if (!projectId.isNullOrBlank()) { - dsl.and(PROJECT_ID.eq(projectId)) + conditions.add(PROJECT_ID.eq(projectId)) } if (messageType != null) { - dsl.and(MESSAGE_TYPE.eq(messageType.name)) + conditions.add(MESSAGE_TYPE.eq(messageType.name)) } if (haveRead != null) { - dsl.and(HAVE_READ.eq(haveRead)) + conditions.add(HAVE_READ.eq(haveRead)) } - return dsl.fetchOne(0, Int::class.java)!! + if (messageId != null) { + conditions.add(MESSAGE_ID.eq(messageId)) + } + return dslContext.selectCount().from(this).where(conditions).fetchOne(0, Int::class.java)!! } } } diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/resources/user/UserStreamUserMessageResourceImpl.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/resources/user/UserStreamUserMessageResourceImpl.kt index dced9fc26e2..3d0796f8d50 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/resources/user/UserStreamUserMessageResourceImpl.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/resources/user/UserStreamUserMessageResourceImpl.kt @@ -45,6 +45,8 @@ class UserStreamUserMessageResourceImpl @Autowired constructor( projectId: String?, messageType: UserMessageType?, haveRead: Boolean?, + messageId: String?, + triggerUserId: String?, page: Int?, pageSize: Int? ): Result> { @@ -54,6 +56,8 @@ class UserStreamUserMessageResourceImpl @Autowired constructor( userId = userId, messageType = messageType, haveRead = haveRead, + messageId = messageId?.ifBlank { null }, + triggerUserId = triggerUserId?.ifBlank { null }, page = page ?: 1, pageSize = pageSize ?: 10 ) diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamGitTransferService.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamGitTransferService.kt index 3b8a1326409..1766853f29a 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamGitTransferService.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamGitTransferService.kt @@ -113,6 +113,15 @@ interface StreamGitTransferService { refreshToken: Boolean? ): Result + /** + * 主动开启git侧 ci + */ + fun enableCi( + userId: String, + projectName: String, + enable: Boolean? = true + ): Result + /** * 获取当前项目的提交记录 */ diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamHistoryService.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamHistoryService.kt index 6791bc9d8ae..8a8fe365f56 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamHistoryService.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamHistoryService.kt @@ -63,6 +63,7 @@ class StreamHistoryService @Autowired constructor( private val channelCode = ChannelCode.GIT + @Suppress("ComplexMethod") fun getHistoryBuildList( userId: String, gitProjectId: Long, @@ -72,6 +73,27 @@ class StreamHistoryService @Autowired constructor( val pageNotNull = search?.page ?: 1 val pageSizeNotNull = search?.pageSize ?: 10 val conf = streamBasicSettingService.getStreamBasicSettingAndCheck(gitProjectId) + + val buildIds = if (!search?.status.isNullOrEmpty()) { + // 如果查询条件有状态信息，需要到引擎里面匹配，拿到buildIds之后再在event build 表里面进行其他条件匹配 + client.get(ServiceBuildResource::class).getBuilds( + userId = userId, + projectId = conf.projectCode!!, + pipelineId = search?.pipelineId, + buildStatus = search?.status, + channelCode = channelCode + ).data + } else null + + if (buildIds?.isEmpty() == true) { + return Page( + page = pageNotNull, + pageSize = pageSizeNotNull, + count = 0, + records = emptyList() + ) + } + val totalPage = gitRequestEventBuildDao.getRequestEventBuildListMultipleCount( dslContext = dslContext, gitProjectId = gitProjectId, @@ -81,8 +103,9 @@ class StreamHistoryService @Autowired constructor( pipelineId = search?.pipelineId, event = search?.event?.map { it.value }?.toSet(), commitMsg = search?.commitMsg, - buildStatus = search?.status?.map { it.name }?.toSet(), - pipelineIds = search?.pipelineIds + buildStatus = null, + pipelineIds = search?.pipelineIds, + buildIds = buildIds?.toSet() ) if (totalPage == 0) { return Page( @@ -102,10 +125,11 @@ class StreamHistoryService @Autowired constructor( pipelineId = search?.pipelineId, event = search?.event?.map { it.value }?.toSet(), commitMsg = search?.commitMsg, - buildStatus = search?.status?.map { it.name }?.toSet(), + buildStatus = null, limit = sqlLimit.limit, offset = sqlLimit.offset, - pipelineIds = search?.pipelineIds + pipelineIds = search?.pipelineIds, + buildIds = buildIds?.toSet() ) val builds = gitRequestBuildList.map { it.buildId }.toSet() logger.info("StreamHistoryService|getHistoryBuildList|builds|$builds") @@ -184,6 +208,7 @@ class StreamHistoryService @Autowired constructor( pageSizeNotNull = pageSize ) } + @Suppress("LongMethod") fun getAllBuildBranchList( userId: String, diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamRequestService.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamRequestService.kt index 413b873a30f..600ef01ffa9 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamRequestService.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamRequestService.kt @@ -81,6 +81,7 @@ class StreamRequestService @Autowired constructor( projectId = projectId, userId = null, messageType = null, + messageId = null, haveRead = null ) val sqlLimit = PageUtil.convertPageSizeToSQLLimit(page = page, pageSize = pageSize) @@ -90,6 +91,7 @@ class StreamRequestService @Autowired constructor( userId = null, messageType = null, haveRead = null, + messageId = null, limit = sqlLimit.limit, offset = sqlLimit.offset ) diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamUserMessageService.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamUserMessageService.kt index 60b2b020ec2..0c620e8d50c 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamUserMessageService.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/StreamUserMessageService.kt @@ -88,6 +88,8 @@ class StreamUserMessageService @Autowired constructor( userId: String, messageType: UserMessageType?, haveRead: Boolean?, + messageId: String?, + triggerUserId: String?, page: Int, pageSize: Int ): Page { @@ -103,8 +105,9 @@ class StreamUserMessageService @Autowired constructor( streamUserMessageDao.getMessageCount( dslContext = dslContext, projectId = projectId, - userId = null, + userId = triggerUserId, messageType = messageType, + messageId = messageId, haveRead = haveRead ) } else { @@ -113,6 +116,7 @@ class StreamUserMessageService @Autowired constructor( projectId = "", userId = userId, messageType = messageType, + messageId = messageId, haveRead = haveRead ) } @@ -130,9 +134,10 @@ class StreamUserMessageService @Autowired constructor( streamUserMessageDao.getMessages( dslContext = dslContext, projectId = projectId, - userId = null, + userId = triggerUserId, messageType = messageType, haveRead = haveRead, + messageId = messageId, limit = sqlLimit.limit, offset = sqlLimit.offset )!! @@ -143,6 +148,7 @@ class StreamUserMessageService @Autowired constructor( userId = userId, messageType = messageType, haveRead = haveRead, + messageId = messageId, limit = sqlLimit.limit, offset = sqlLimit.offset )!! diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/transfer/StreamGithubTransferService.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/transfer/StreamGithubTransferService.kt index cdc0519d47e..ac4fe644961 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/transfer/StreamGithubTransferService.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/transfer/StreamGithubTransferService.kt @@ -255,6 +255,11 @@ class StreamGithubTransferService @Autowired constructor( return Result(AuthorizeResult(HTTP_200)) } + override fun enableCi(userId: String, projectName: String, enable: Boolean?): Result { + // github 不支持 + return Result(true) + } + override fun getCommits( userId: String, gitProjectId: Long, diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/transfer/StreamTGitTransferService.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/transfer/StreamTGitTransferService.kt index c2b356119cb..775c57184ec 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/transfer/StreamTGitTransferService.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/service/transfer/StreamTGitTransferService.kt @@ -215,6 +215,15 @@ class StreamTGitTransferService @Autowired constructor( ) } + override fun enableCi(userId: String, projectName: String, enable: Boolean?): Result { + return client.get(ServiceGitResource::class).enableCi( + projectName = projectName, + token = getAndCheckOauthToken(userId).accessToken, + tokenType = TokenTypeEnum.OAUTH, + enable = enable + ) + } + override fun getCommits( userId: String, gitProjectId: Long, diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/StreamYamlBuild.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/StreamYamlBuild.kt index 9924d5553a6..83ce7aeff4b 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/StreamYamlBuild.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/StreamYamlBuild.kt @@ -55,6 +55,7 @@ import com.tencent.devops.stream.config.StreamGitConfig import com.tencent.devops.stream.dao.GitPipelineResourceDao import com.tencent.devops.stream.pojo.StreamDeleteEvent import com.tencent.devops.stream.pojo.enums.TriggerReason +import com.tencent.devops.stream.service.StreamGitService import com.tencent.devops.stream.trigger.actions.BaseAction import com.tencent.devops.stream.trigger.actions.data.StreamTriggerPipeline import com.tencent.devops.stream.trigger.actions.data.isStreamMr @@ -87,6 +88,7 @@ class StreamYamlBuild @Autowired constructor( private val redisOperation: RedisOperation, private val streamTimerService: StreamTimerService, private val deleteEventService: DeleteEventService, + private val streamGitService: StreamGitService, private val streamTriggerCache: StreamTriggerCache, private val repoTriggerEventService: RepoTriggerEventService, private val pipelineResourceDao: GitPipelineResourceDao, @@ -499,6 +501,10 @@ class StreamYamlBuild @Autowired constructor( job.runsOn.poolName = getEnvName(action, job.runsOn.poolName, yaml.resource?.pools) } } + // 替换finally中的构建机 + yaml.finally?.forEach { fina -> + fina.runsOn.poolName = getEnvName(action, fina.runsOn.poolName, yaml.resource?.pools) + } return yaml } @@ -509,29 +515,25 @@ class StreamYamlBuild @Autowired constructor( pools.filter { !it.from.isNullOrBlank() && !it.name.isNullOrBlank() }.forEach label@{ if (it.name == poolName) { - try { - val repoNameAndPool = it.from!!.split("@") - if (repoNameAndPool.size != 2 || repoNameAndPool[0].isBlank() || repoNameAndPool[1].isBlank()) { - return@label - } - - val gitProjectInfo = streamTriggerCache.getAndSaveRequestGitProjectInfo( - gitProjectKey = repoNameAndPool[0], - action = action, - getProjectInfo = action.api::getGitProjectInfo - )!! + val repoNameAndPool = it.from!!.split("@") + if (repoNameAndPool.size != 2 || repoNameAndPool[0].isBlank() || repoNameAndPool[1].isBlank()) { + return@label + } - val result = GitCommonUtils.getCiProjectId( - "${gitProjectInfo.gitProjectId}@${repoNameAndPool[1]}", - streamGitConfig.getScmType() + val gitProjectInfo = streamGitService.getProjectInfo(repoNameAndPool[0]) + ?: throw StreamTriggerException( + action, + TriggerReason.PIPELINE_PREPARE_ERROR, + listOf("跨项目引用第三方构建资源池错误: 获取远程仓库(${repoNameAndPool[0]})信息失败, 请检查填写是否正确") ) - logger.info("StreamYamlBuild|getEnvName|envName|$result") - return result - } catch (e: Exception) { - logger.warn("StreamYamlBuild|getEnvName|$poolName|error", e) - return poolName - } + val result = GitCommonUtils.getCiProjectId( + "${gitProjectInfo.gitProjectId}@${repoNameAndPool[1]}", + streamGitConfig.getScmType() + ) + + logger.info("StreamYamlBuild|getEnvName|envName|$result") + return result } } logger.info("StreamYamlBuild|getEnvName|no match. envName|$poolName") diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/actions/streamActions/StreamRepoTriggerAction.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/actions/streamActions/StreamRepoTriggerAction.kt index f5fba86355e..e6fcc9a5c63 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/actions/streamActions/StreamRepoTriggerAction.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/actions/streamActions/StreamRepoTriggerAction.kt @@ -78,7 +78,6 @@ class StreamRepoTriggerAction( override fun checkAndDeletePipeline(path2PipelineExists: Map) {} override fun getYamlPathList(): List { - val changeSet = getChangeSet() return GitActionCommon.getYamlPathList( action = baseAction, gitProjectId = getGitProjectIdOrName(), @@ -110,7 +109,9 @@ class StreamRepoTriggerAction( return baseAction.isMatch(triggerOn) } - override fun getUserVariables(yamlVariables: Map?): Map? = null + override fun getUserVariables( + yamlVariables: Map? + ): Map? = baseAction.getUserVariables(yamlVariables) override fun needSaveOrUpdateBranch() = false diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/components/SendCommitCheck.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/components/SendCommitCheck.kt index ddc6fec6f08..a456686fc49 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/components/SendCommitCheck.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/components/SendCommitCheck.kt @@ -44,13 +44,13 @@ import com.tencent.devops.stream.trigger.actions.data.context.isSuccess import com.tencent.devops.stream.trigger.actions.data.isStreamMr import com.tencent.devops.stream.trigger.parsers.StreamTriggerCache import com.tencent.devops.stream.util.StreamPipelineUtils -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component import java.time.Duration import java.time.Instant import java.time.LocalDateTime import java.time.ZoneId +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component @Suppress("NestedBlockDepth") @Component @@ -64,7 +64,7 @@ class SendCommitCheck @Autowired constructor( private const val BUILD_RUNNING_DESC = "Running." private const val BUILD_STAGE_SUCCESS_DESC = "Warning: your pipeline「%s」 is stage succeed. Rejected by %s, reason is %s." - private const val BUILD_SUCCESS_DESC = "Successful in %sm." + private const val BUILD_SUCCESS_DESC = "Successful in %s." private const val BUILD_CANCEL_DESC = "Your pipeline「%s」 was cancelled." private const val BUILD_FAILED_DESC = "Failing after %sm." private const val BUILD_GATE_REVIEW_DESC = @@ -100,7 +100,7 @@ class SendCommitCheck @Autowired constructor( gitProjectName = streamGitProjectInfo.name, state = finishData.getGitCommitCheckState(), block = action.metaData.isStreamMr() && action.data.setting.enableMrBlock && - !finishData.isSuccess(), + !finishData.isSuccess(), context = "${action.data.context.pipeline!!.filePath}@${action.metaData.streamObjectKind.name}", targetUrl = getTargetUrl(action), description = getDescByBuildStatus( @@ -159,21 +159,33 @@ class SendCommitCheck @Autowired constructor( val (name, reason) = getReviewInfo(finishData) BUILD_STAGE_SUCCESS_DESC.format(pipelineName, name, reason) } else { - BUILD_SUCCESS_DESC.format(getFinishTime(finishData.startTime).toString()) + BUILD_SUCCESS_DESC.format(getFinishTime(finishData.startTime)) } } finishData.getBuildStatus().isCancel() -> { BUILD_CANCEL_DESC.format(pipelineName) } else -> { - BUILD_FAILED_DESC.format(getFinishTime(finishData.startTime).toString()) + BUILD_FAILED_DESC.format(getFinishTime(finishData.startTime)) } } - private fun getFinishTime(startTimeTimeStamp: Long?): Long { + private fun getFinishTime(startTimeTimeStamp: Long?): String { val zoneId = ZoneId.systemDefault() val startTime = LocalDateTime.ofInstant(startTimeTimeStamp?.let { Instant.ofEpochMilli(it) }, zoneId) - return startTime.between(LocalDateTime.now()).toMinutes() + return startTime.between(LocalDateTime.now()).format() + } + + private fun Duration.format(): String { + if (this === Duration.ZERO) { + return "0s" + } + val day = (seconds / 86400).toInt() + val hours = ((seconds / 3600) % 24).toInt() + val minutes = (seconds % 3600 / 60).toInt() + val secs = (seconds % 60).toInt() + fun join(int: Int, name: String) = if (int > 0) " $int$name" else "" + return join(day, "d") + join(hours, "h") + join(minutes, "m") + join(secs, "s") } private fun getTargetUrl( diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/components/SendNotify.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/components/SendNotify.kt index 3a534161d7d..e80c57fea5d 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/components/SendNotify.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/components/SendNotify.kt @@ -44,7 +44,6 @@ import com.tencent.devops.stream.config.StreamGitConfig import com.tencent.devops.stream.trigger.actions.BaseAction import com.tencent.devops.stream.trigger.actions.data.context.BuildFinishData import com.tencent.devops.stream.trigger.actions.data.context.getBuildStatus -import com.tencent.devops.stream.trigger.actions.data.context.getGitCommitCheckState import com.tencent.devops.stream.trigger.actions.data.context.isSuccess import com.tencent.devops.stream.trigger.actions.data.isStreamMr import com.tencent.devops.stream.trigger.actions.streamActions.StreamMrAction @@ -157,12 +156,12 @@ class SendNotify @Autowired constructor( if (realReceivers.isEmpty()) { realReceivers = mutableSetOf(action.data.eventCommon.userId) } - val state = action.data.context.finishData!!.getGitCommitCheckState() + val status = action.data.context.finishData!!.getBuildStatus() when (notifyType) { StreamNotifyType.EMAIL -> { val request = SendEmail.getEmailSendRequest( - state = state, + status = status, receivers = realReceivers, projectName = projectName, branchName = branchName, @@ -187,7 +186,7 @@ class SendNotify @Autowired constructor( val (rtxReceivers, receiversType) = Pair(realReceivers, ReceiverType.SINGLE) SendRtx.getRtxSendRequest( - state = state, + status = status, receivers = rtxReceivers, projectName = projectName, branchName = branchName, @@ -262,11 +261,9 @@ class SendNotify @Autowired constructor( val vars = mutableSetOf() value.forEach { re -> - if (!re.contains(',')) { - vars.add(re) - return@forEach - } - vars.addAll(re.split(",").asSequence().filter { it.isNotBlank() }.map { it.trim() }.toSet()) + vars.addAll(re.parseEnv(variables) + .split(",").asSequence().filter { it.isNotBlank() }.map { it.trim() }.toSet() + ) } if (variables.isNullOrEmpty()) { @@ -277,6 +274,9 @@ class SendNotify @Autowired constructor( }.toSet() } + private fun String.parseEnv(data: Map?) = + if (!data.isNullOrEmpty()) EnvUtils.parseEnv(this, data) else this + private fun getNoticeType(buildId: String, type: String?): StreamNotifyType? { return when (type) { "email" -> { diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/SendEmail.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/SendEmail.kt index 2775deaa76e..4108e74e37c 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/SendEmail.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/SendEmail.kt @@ -29,16 +29,16 @@ package com.tencent.devops.stream.trigger.listener.notify import com.tencent.devops.common.api.util.DateTimeUtil import com.tencent.devops.common.notify.enums.NotifyType +import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.notify.pojo.SendNotifyMessageTemplateRequest import com.tencent.devops.process.pojo.BuildHistory -import com.tencent.devops.stream.trigger.pojo.enums.StreamCommitCheckState import com.tencent.devops.stream.trigger.pojo.enums.StreamNotifyTemplateEnum import com.tencent.devops.stream.util.StreamPipelineUtils import java.util.Date object SendEmail { fun getEmailSendRequest( - state: StreamCommitCheckState, + status: BuildStatus, receivers: Set, ccs: MutableSet?, projectName: String, @@ -52,12 +52,11 @@ object SendEmail { streamUrl: String, gitProjectId: String ): SendNotifyMessageTemplateRequest { - val isSuccess = state == StreamCommitCheckState.SUCCESS val titleParams = mapOf( "title" to ( if (title.isNullOrBlank()) { V2NotifyTemplate.getEmailTitle( - isSuccess = isSuccess, + status = status, projectName = projectName, branchName = branchName, pipelineName = pipelineName, buildNum = build.buildNum.toString() @@ -71,7 +70,7 @@ object SendEmail { "content" to ( if (content.isNullOrBlank()) { V2NotifyTemplate.getEmailContent( - isSuccess = isSuccess, + status = status, projectName = projectName, branchName = branchName, pipelineName = pipelineName, diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/SendRtx.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/SendRtx.kt index 78f22cc185d..95726aa2db9 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/SendRtx.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/SendRtx.kt @@ -30,15 +30,15 @@ package com.tencent.devops.stream.trigger.listener.notify import com.tencent.devops.common.api.enums.ScmType import com.tencent.devops.common.api.util.DateTimeUtil import com.tencent.devops.common.notify.enums.NotifyType +import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.notify.pojo.SendNotifyMessageTemplateRequest import com.tencent.devops.process.pojo.BuildHistory -import com.tencent.devops.stream.trigger.pojo.enums.StreamCommitCheckState import com.tencent.devops.stream.trigger.pojo.enums.StreamNotifyTemplateEnum import com.tencent.devops.stream.util.StreamPipelineUtils object SendRtx { fun getRtxSendRequest( - state: StreamCommitCheckState, + status: BuildStatus, receivers: Set, projectName: String, branchName: String, @@ -55,14 +55,13 @@ object SendRtx { gitProjectId: String, scmType: ScmType ): SendNotifyMessageTemplateRequest { - val isSuccess = state == StreamCommitCheckState.SUCCESS val titleParams = mapOf( "title" to "" ) val bodyParams = mapOf( "content" to if (content.isNullOrBlank()) { getRtxCustomContent( - isSuccess = isSuccess, + status = status, projectName = projectName, branchName = branchName, pipelineName = pipelineName, @@ -78,7 +77,7 @@ object SendRtx { ) } else { getRtxCustomUserContent( - isSuccess = isSuccess, + status = status, gitProjectId = gitProjectId, pipelineId = pipelineId, build = build, @@ -99,18 +98,19 @@ object SendRtx { // 为用户的内容增加链接 private fun getRtxCustomUserContent( - isSuccess: Boolean, + status: BuildStatus, gitProjectId: String, pipelineId: String, build: BuildHistory, content: String, streamUrl: String ): String { - val state = if (isSuccess) { - Triple("✔", "info", "success") - } else { - Triple("❌", "warning", "failed") + val state = when { + status.isSuccess() -> Triple("✔", "info", "success") + status.isCancel() -> Triple("❕", "warning", "cancel") + else -> Triple("❌", "warning", "failed") } + val detailUrl = StreamPipelineUtils.genStreamV2BuildUrl( homePage = streamUrl, gitProjectId = gitProjectId, @@ -121,7 +121,7 @@ object SendRtx { } private fun getRtxCustomContent( - isSuccess: Boolean, + status: BuildStatus, projectName: String, branchName: String, pipelineName: String, @@ -135,35 +135,36 @@ object SendRtx { streamUrl: String, gitProjectId: String ): String { - val state = if (isSuccess) { - Triple("✔", "info", "success") - } else { - Triple("❌", "warning", "failed") + val state = when { + status.isSuccess() -> Triple("✔", "info", "success") + status.isCancel() -> Triple("❕", "warning", "cancel") + else -> Triple("❌", "warning", "failed") } + val request = if (isMr) { "Merge requests [[!$requestId]]($gitUrl/$projectName/merge_requests/$requestId)" + - "opened by $openUser \n" + "opened by $openUser \n" } else { if (requestId.length >= 8) { "Commit [[${requestId.subSequence(0, 8)}]]($gitUrl/$projectName/commit/$requestId)" + - "pushed by $openUser \n" + "pushed by $openUser \n" } else { "Manual Triggered by $openUser \n" } } val costTime = "Time cost ${DateTimeUtil.formatMillSecond(buildTime ?: 0)}. \n " return " ${state.first} " + - "$projectName($branchName) - $pipelineName #${build.buildNum} run ${state.third} \n " + - request + - costTime + - "[查看详情]" + - "(${ - StreamPipelineUtils.genStreamV2BuildUrl( - homePage = streamUrl, - gitProjectId = gitProjectId, - pipelineId = pipelineId, - buildId = build.id - ) - })" + "$projectName($branchName) - $pipelineName #${build.buildNum} run ${state.third} \n " + + request + + costTime + + "[查看详情]" + + "(${ + StreamPipelineUtils.genStreamV2BuildUrl( + homePage = streamUrl, + gitProjectId = gitProjectId, + pipelineId = pipelineId, + buildId = build.id + ) + })" } } diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/V2NotifyTemplate.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/V2NotifyTemplate.kt index d42fb4c0c20..2d01ba8912a 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/V2NotifyTemplate.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/listener/notify/V2NotifyTemplate.kt @@ -27,20 +27,22 @@ package com.tencent.devops.stream.trigger.listener.notify +import com.tencent.devops.common.pipeline.enums.BuildStatus + // v2 Stream的默认通知模板 object V2NotifyTemplate { fun getEmailTitle( - isSuccess: Boolean, + status: BuildStatus, projectName: String, branchName: String, pipelineName: String, buildNum: String ): String { - val state = if (isSuccess) { - "run successes" - } else { - "run failed" + val state = when { + status.isSuccess() -> "run successes" + status.isCancel() -> "run cancel" + else -> "run failed" } return """ [$projectName][$branchName] $pipelineName #$buildNum $state @@ -48,7 +50,7 @@ object V2NotifyTemplate { } fun getEmailContent( - isSuccess: Boolean, + status: BuildStatus, projectName: String, branchName: String, pipelineName: String, @@ -59,10 +61,10 @@ object V2NotifyTemplate { commitId: String, webUrl: String ): String { - val state = if (isSuccess) { - "run successes" - } else { - "run failed" + val state = when { + status.isSuccess() -> "run successes" + status.isCancel() -> "run cancel" + else -> "run failed" } return """ diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/parsers/triggerMatch/TriggerMatcher.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/parsers/triggerMatch/TriggerMatcher.kt index 310ce00e6b6..c9ab08ab1a3 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/parsers/triggerMatch/TriggerMatcher.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/parsers/triggerMatch/TriggerMatcher.kt @@ -342,6 +342,13 @@ class TriggerMatcher @Autowired constructor( return TriggerBody().triggerFail("on.tag.users-ignore", "trigger user($userId) match") } + if (fromBranch == null && !tagRule.fromBranches.isNullOrEmpty()) { + return TriggerBody().triggerFail( + "on.tag.from-branches", + "client push tag not support from-branches" + ) + } + if (fromBranch != null && !BranchMatchUtils.isBranchMatch(tagRule.fromBranches, fromBranch)) { return TriggerBody().triggerFail( "on.tag.from-branches", diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/service/StreamEventService.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/service/StreamEventService.kt index 9d8841b1b92..234513d81a9 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/service/StreamEventService.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/trigger/service/StreamEventService.kt @@ -111,7 +111,8 @@ class StreamEventService @Autowired constructor( block = setting.enableMrBlock && commitCheckBlock, targetUrl = StreamPipelineUtils.genStreamV2NotificationsUrl( streamUrl = streamGitConfig.streamUrl ?: throw ParamBlankException("启动配置缺少 streamGitConfig"), - gitProjectId = getGitProjectId() + gitProjectId = getGitProjectId(), + messageId = action.data.context.requestEventId.toString() ), context = "${context.pipeline!!.filePath}@${action.metaData.streamObjectKind.name}", description = TriggerReason.getTriggerReason(reason)?.summary ?: reason, diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/util/RetryUtils.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/util/RetryUtils.kt index acd88bd2029..9eed1947fde 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/util/RetryUtils.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/util/RetryUtils.kt @@ -28,8 +28,11 @@ package com.tencent.devops.stream.util import com.tencent.devops.common.api.exception.ClientException +import com.tencent.devops.common.api.exception.RemoteServiceException +import org.slf4j.LoggerFactory object RetryUtils { + private val logger = LoggerFactory.getLogger(RetryUtils::class.java) @Throws(ClientException::class) fun clientRetry(retryTime: Int = 5, retryPeriodMills: Long = 500, action: () -> T): T { @@ -43,6 +46,16 @@ object RetryUtils { Thread.sleep(retryPeriodMills) } clientRetry(action = action, retryTime = retryTime - 1, retryPeriodMills = retryPeriodMills) + } catch (e: RemoteServiceException) { + // 对限流重试 + if (e.httpStatus != 429) throw e + if (retryTime - 1 < 0) { + throw e + } + logger.info("Remote service return 429 and message:${e.message}") + // 固定延迟1s + Thread.sleep(1000) + clientRetry(action = action, retryTime = retryTime - 1) } } diff --git a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/util/StreamPipelineUtils.kt b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/util/StreamPipelineUtils.kt index 532b08f5650..1f288890065 100644 --- a/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/util/StreamPipelineUtils.kt +++ b/src/backend/ci/core/stream/biz-stream/src/main/kotlin/com/tencent/devops/stream/util/StreamPipelineUtils.kt @@ -56,7 +56,11 @@ object StreamPipelineUtils { return "$url/#$gitProjectId" } - fun genStreamV2NotificationsUrl(streamUrl: String, gitProjectId: String) = "$streamUrl/notifications#$gitProjectId" + fun genStreamV2NotificationsUrl( + streamUrl: String, + gitProjectId: String, + messageId: String + ) = "$streamUrl/notifications?id=$messageId#$gitProjectId" fun createEmptyPipelineAndSetting(displayName: String) = PipelineModelAndSetting( model = Model( diff --git a/src/backend/ci/core/worker/worker-agent/src/main/kotlin/com/tencent/devops/agent/Application.kt b/src/backend/ci/core/worker/worker-agent/src/main/kotlin/com/tencent/devops/agent/Application.kt index 69aafaf85a1..ba09ff4bcd1 100644 --- a/src/backend/ci/core/worker/worker-agent/src/main/kotlin/com/tencent/devops/agent/Application.kt +++ b/src/backend/ci/core/worker/worker-agent/src/main/kotlin/com/tencent/devops/agent/Application.kt @@ -34,8 +34,6 @@ import com.tencent.devops.common.api.enums.EnumLoader import com.tencent.devops.common.api.util.DHUtil import com.tencent.devops.common.api.util.OkhttpUtils import com.tencent.devops.common.pipeline.ElementSubTypeRegisterLoader -import com.tencent.devops.worker.common.AGENT_ID -import com.tencent.devops.worker.common.AGENT_SECRET_KEY import com.tencent.devops.worker.common.BUILD_TYPE import com.tencent.devops.worker.common.Runner import com.tencent.devops.worker.common.WorkspaceInterface @@ -150,9 +148,10 @@ private fun doResponse( val buildLessTask: Map = jacksonObjectMapper().readValue(responseBody) buildLessTask.forEach { (t, u) -> when (t) { - "agentId" -> System.setProperty(AGENT_ID, u) - "secretKey" -> System.setProperty(AGENT_SECRET_KEY, u) - "projectId" -> System.setProperty("devops_project_id", u) + "agentId" -> DockerEnv.setAgentId(u) + "secretKey" -> DockerEnv.setAgentSecretKey(u) + "projectId" -> DockerEnv.setProjectId(u) + "buildId" -> DockerEnv.setBuildId(u) } } true diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Constants.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Constants.kt index 57c87a8b6ca..467ab8957f5 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Constants.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Constants.kt @@ -31,10 +31,6 @@ const val BUILD_ID = "devops.build.id" const val BUILD_TYPE = "build.type" -const val AGENT_ID = "devops_agent_id" - -const val AGENT_SECRET_KEY = "devops_agent_secret_key" - const val WORKSPACE_ENV = "WORKSPACE" const val COMMON_ENV_CONTEXT = "common_env" diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt index 8fe5cd5487b..9c141fb8ea8 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt @@ -43,6 +43,7 @@ import com.tencent.devops.process.utils.PIPELINE_RETRY_COUNT import com.tencent.devops.process.utils.PipelineVarUtil import com.tencent.devops.worker.common.env.BuildEnv import com.tencent.devops.worker.common.env.BuildType +import com.tencent.devops.worker.common.env.DockerEnv import com.tencent.devops.worker.common.heartbeat.Heartbeat import com.tencent.devops.worker.common.logger.LoggerService import com.tencent.devops.worker.common.service.EngineService @@ -69,8 +70,7 @@ object Runner { logger.info("Start the worker ...") ErrorMsgLogUtil.init() var workspacePathFile: File? = null - // 启动成功, 报告process我已经启动了, #1613 如果这都失败了，则也无法向后台上报信息了。将由devopsAgent监控传递 - val buildVariables = EngineService.setStarted() + val buildVariables = getBuildVariables() var failed = false try { BuildEnv.setBuildId(buildVariables.buildId) @@ -89,7 +89,7 @@ object Runner { } finally { LoggerService.stop() LoggerService.archiveLogFiles() - EngineService.endBuild(buildVariables) + EngineService.endBuild(buildVariables.variables) QuotaService.removeRunningAgent(buildVariables) Heartbeat.stop() } @@ -118,7 +118,7 @@ object Runner { errorCode = ErrorCode.SYSTEM_WORKER_INITIALIZATION_ERROR ) ) - EngineService.endBuild(buildVariables) + EngineService.endBuild(buildVariables.variables) throw ignore } finally { finally(workspacePathFile, failed) @@ -129,6 +129,22 @@ object Runner { } } + private fun getBuildVariables(): BuildVariables { + try { + // 启动成功, 报告process我已经启动了 + return EngineService.setStarted() + } catch (e: Exception) { + logger.warn("Set started catch unknown exceptions", e) + // 启动失败，尝试结束构建 + try { + EngineService.endBuild(emptyMap(), DockerEnv.getBuildId()) + } catch (e: Exception) { + logger.warn("End build catch unknown exceptions", e) + } + throw e + } + } + private fun prepareWorkspace(buildVariables: BuildVariables, workspaceInterface: WorkspaceInterface): File { // 为进程加上ShutdownHook事件 KillBuildProcessTree.addKillProcessTreeHook( diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/engine/EngineBuildSDKApi.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/engine/EngineBuildSDKApi.kt index 373f54d4203..7da07c2c5c2 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/engine/EngineBuildSDKApi.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/engine/EngineBuildSDKApi.kt @@ -45,7 +45,7 @@ interface EngineBuildSDKApi : WorkerRestApiSDK { fun completeTask(result: BuildTaskResult, retryCount: Int): Result - fun endTask(buildVariables: BuildVariables, retryCount: Int): Result + fun endTask(variables: Map, envBuildId: String, retryCount: Int): Result fun heartbeat(executeCount: Int = 1): Result diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/engine/impl/EngineBuildResourceApi.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/engine/impl/EngineBuildResourceApi.kt index cd97cd230de..093b8a0ca87 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/engine/impl/EngineBuildResourceApi.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/engine/impl/EngineBuildResourceApi.kt @@ -98,7 +98,11 @@ open class EngineBuildResourceApi : AbstractBuildResourceApi(), EngineBuildSDKAp return objectMapper.readValue(responseContent) } - override fun endTask(buildVariables: BuildVariables, retryCount: Int): Result { + override fun endTask(variables: Map, envBuildId: String, retryCount: Int): Result { + if (envBuildId.isNotBlank()) { + buildId = envBuildId + } + val path = getRequestUrl(path = "api/build/worker/end", retryCount = retryCount) val request = buildPost(path) val errorMessage = "构建完成请求失败" diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/env/DockerEnv.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/env/DockerEnv.kt index 25fbf9258ba..cde01cbbd8d 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/env/DockerEnv.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/env/DockerEnv.kt @@ -34,6 +34,7 @@ object DockerEnv { private val logger = LoggerFactory.getLogger(DockerEnv::class.java) private const val PROJECT_ID = "devops_project_id" + private const val BUILD_ID = "devops_build_id" private const val AGENT_ID = "devops_agent_id" private const val AGENT_SECRET_KEY = "devops_agent_secret_key" private const val AGENT_GATEWAY = "devops_gateway" @@ -43,14 +44,19 @@ object DockerEnv { private const val HOSTNAME = "HOSTNAME" private var projectId: String? = null + private var buildId: String? = null private var agentId: String? = null private var secretKey: String? = null private var gateway: String? = null private var jobPool: String? = null - private var dokerHostIp: String? = null + private var dockerHostIp: String? = null private var dockerHostPort: String? = null private var hostname: String? = null + fun setProjectId(projectId: String) { + System.setProperty(PROJECT_ID, projectId) + } + fun getProjectId(): String { if (projectId.isNullOrBlank()) { synchronized(this) { @@ -66,6 +72,29 @@ object DockerEnv { return projectId!! } + fun setBuildId(buildId: String) { + System.setProperty(BUILD_ID, buildId) + } + + fun getBuildId(): String { + if (buildId.isNullOrBlank()) { + synchronized(this) { + if (buildId.isNullOrBlank()) { + buildId = getProperty(BUILD_ID) + if (buildId.isNullOrBlank()) { + throw PropertyNotExistException(BUILD_ID, "Empty build Id") + } + logger.info("Get the build ID($buildId)") + } + } + } + return buildId!! + } + + fun setAgentId(agentId: String) { + System.setProperty(AGENT_ID, agentId) + } + fun getAgentId(): String { if (agentId.isNullOrBlank()) { synchronized(this) { @@ -81,6 +110,10 @@ object DockerEnv { return agentId!! } + fun setAgentSecretKey(secretKey: String) { + System.setProperty(AGENT_SECRET_KEY, secretKey) + } + fun getAgentSecretKey(): String { if (secretKey.isNullOrBlank()) { synchronized(this) { @@ -128,18 +161,18 @@ object DockerEnv { } fun getDockerHostIp(): String { - if (dokerHostIp.isNullOrBlank()) { + if (dockerHostIp.isNullOrBlank()) { synchronized(this) { - if (dokerHostIp.isNullOrBlank()) { - dokerHostIp = getProperty(DOCKER_HOST_IP) - if (dokerHostIp.isNullOrBlank()) { + if (dockerHostIp.isNullOrBlank()) { + dockerHostIp = getProperty(DOCKER_HOST_IP) + if (dockerHostIp.isNullOrBlank()) { throw PropertyNotExistException(DOCKER_HOST_IP, "Empty dokerHostIp") } - logger.info("Get the dokerHostIp($dokerHostIp)") + logger.info("Get the dokerHostIp($dockerHostIp)") } } } - return dokerHostIp!! + return dockerHostIp!! } fun getDockerHostPort(): String { diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/EngineService.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/EngineService.kt index 28dbc349f72..0e21826d6a2 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/EngineService.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/EngineService.kt @@ -110,14 +110,14 @@ object EngineService { } } - fun endBuild(buildVariables: BuildVariables) { + fun endBuild(variables: Map, buildId: String = "") { var retryCount = 0 val result = HttpRetryUtils.retry { if (retryCount > 0) { logger.warn("retry|time=$retryCount|endBuild") sleepInterval(retryCount) } - buildApi.endTask(buildVariables, retryCount++) + buildApi.endTask(variables, buildId, retryCount++) } if (result.isNotOk()) { throw RemoteServiceException("Failed to end build task") diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/utils/ShellUtil.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/utils/ShellUtil.kt index da688e6ae8b..c61e29b133a 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/utils/ShellUtil.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/utils/ShellUtil.kt @@ -37,6 +37,7 @@ import com.tencent.devops.worker.common.logger.LoggerService import com.tencent.devops.worker.common.task.script.ScriptEnvUtils import java.io.File import java.nio.file.Files +import java.util.regex.Pattern @Suppress("ALL") object ShellUtil { @@ -78,6 +79,7 @@ object ShellUtil { private val specialKey = listOf(".", "-") private val specialCharToReplace = Regex("['\n]") // --bug=75509999 Agent环境变量中替换掉破坏性字符 + private const val chineseRegex = "[\u4E00-\u9FA5|\\！|\\，|\\。|\\（|\\）|\\《|\\》|\\“|\\”|\\？|\\：|\\；|\\【|\\】]" fun execute( buildId: String, @@ -231,6 +233,15 @@ object ShellUtil { } private fun specialEnv(key: String): Boolean { - return specialKey.any { key.contains(it) } + return specialKey.any { key.contains(it) } || isContainChinese(key) + } + + private fun isContainChinese(str: String): Boolean { + val pattern = Pattern.compile(chineseRegex) + val matcher = pattern.matcher(str) + if (matcher.find()) { + return true + } + return false } } diff --git a/src/backend/ci/gradle.properties b/src/backend/ci/gradle.properties index ba52b9d1140..359d8247c04 100644 --- a/src/backend/ci/gradle.properties +++ b/src/backend/ci/gradle.properties @@ -40,4 +40,5 @@ org.gradle.daemon=true org.gradle.configureondemand=false org.gradle.parallel=true org.gradle.caching=true -org.gradle.jvmargs=-Xms512M -Xmx5120M +org.gradle.jvmargs=-Xms512M -Xmx4096M +org.gradle.daemon.idletimeout=300000 diff --git a/src/backend/ci/settings.gradle.kts b/src/backend/ci/settings.gradle.kts index 4ee340d133e..a2a27e36a1e 100644 --- a/src/backend/ci/settings.gradle.kts +++ b/src/backend/ci/settings.gradle.kts @@ -30,7 +30,6 @@ rootProject.name = "bk-ci-backend" // 适用于project的plugins pluginManagement { repositories { - mavenLocal() if (System.getenv("GITHUB_WORKFLOW") == null) { // 普通环境 maven(url = "https://mirrors.tencent.com/nexus/repository/maven-public") maven(url = "https://mirrors.tencent.com/nexus/repository/gradle-plugins/") @@ -45,6 +44,7 @@ pluginManagement { mavenCentral() gradlePluginPortal() } + mavenLocal() } } @@ -73,6 +73,7 @@ include(":core:common:common-stream") include(":core:common:common-expression") include(":core:common:common-test") include(":core:common:common-auth") +include(":core:common:common-kubernetes") include(":core:common:common-auth:common-auth-api") include(":core:common:common-auth:common-auth-mock") include(":core:common:common-auth:common-auth-blueking") diff --git a/src/backend/dispatch-k8s-manager/README.md b/src/backend/dispatch-k8s-manager/README.md index 52508562503..402583940cf 100644 --- a/src/backend/dispatch-k8s-manager/README.md +++ b/src/backend/dispatch-k8s-manager/README.md @@ -1,15 +1,15 @@ # kubernetes-manager -### 开发须知 +## 开发须知 1. 修改resource下的config文件时需要同步修改 manifests中的configmap，保持一致。 2. 修改接口后，需要运行 ./swagger/init-swager.sh 重新初始化swagger文档。 -### 使用须知 +## 使用须知 kubernetes-manager可以使用二进制方式启动，也可以使用容器方式（更加推荐作为容器启动）。 -#### 以容器方式启动 +### 以容器方式启动 1. 打包镜像。通过修改 makefile 中的 LOCAL_REGISTR与LOCAL_IMAGE，修改默认镜像参数后 make -f ./Makefile image.xxx 打包自己需要的架构。或者直接使用docker文件夹下Dockerfile参考makefile中命令自行打包。打包后即可作为docker容器使用（需配合现有的redis和mysql）。 @@ -20,10 +20,22 @@ kubernetes-manager可以使用二进制方式启动，也可以使用容器方 - **登录调试相关** 因为登录调试需要将https链接转为wss与kuberntes通信，所以需要 **指定需要登录调试集群的kubeconfig**，指定方式参考 **如何链接不同的kubernetes集群**。 - **realResource优化** 优化使用了kubernetes-scheduler-pluign和prometheus的特性，所以需要配置 prometheus同时需要安装 [ci-dispatch-k8s-manager-plugin](https://github.com/TencentBlueKing/ci-dispatch-k8s-manager-plugin) 插件。 -#### 以二进制的方式启动 +#### kubernetes-manager和bk-ci同k8s集群同namespace部署 +配置bk-ci helm values +'bkCiKubernetesHost': "http://kubernetes-manager" // 默认kubernetes-manager的service类型为 NodePort +'bkCiKubernetesToken': "landun" // 同kubernetesManager.apiserver.auth.apiToken.value配置 +#### kubernetes-manager和bk-ci同集群不同namespace部署 +配置bk-ci helm values +'bkCiKubernetesHost': "http://kubernetes-manager.{{ .Release.Name }}" // 默认kubernetes-manager的service类型为 NodePort +'bkCiKubernetesToken': "landun" // 同kubernetesManager.apiserver.auth.apiToken.value配置 +#### kubernetes-manager和bk-ci不同集群部署 +配置bk-ci helm values +'bkCiKubernetesHost': "http://node:port" // // 默认kubernetes-manager的service类型为 NodePort +'bkCiKubernetesToken': "landun" // 同kubernetesManager.apiserver.auth.apiToken.value配置 + +### 以二进制的方式启动 1. 打包二进制。参考makefile中的 build.xxx 和 release.xxx 同时修改makefile中 CONFIG_DIR，OUT_DIR来存放配置文件和目录文件（配置文件格式可参考 resources 目录）。 2. 补充说明： - 二进制格式启动类似直接镜像启动，可以相互参考。同时二进制格式启动一样不具备mysql和redis，需要自行准备。 - diff --git a/src/backend/dispatch-k8s-manager/manifests/chart/Chart.yaml b/src/backend/dispatch-k8s-manager/manifests/chart/Chart.yaml index 5b9395c3812..ccd88943079 100644 --- a/src/backend/dispatch-k8s-manager/manifests/chart/Chart.yaml +++ b/src/backend/dispatch-k8s-manager/manifests/chart/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubernetes-manager description: A Helm chart for BlueKing CI Kubernetes Manager type: application -version: 0.0.34 +version: 0.0.36 appVersion: 0.0.31 home: https://github.com/Tencent/bk-ci @@ -14,4 +14,4 @@ dependencies: - name: redis version: 14.8.8 repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file + condition: redis.enabled diff --git a/src/backend/dispatch-k8s-manager/manifests/chart/kubernetes-manager-0.0.34.tgz b/src/backend/dispatch-k8s-manager/manifests/chart/kubernetes-manager-0.0.34.tgz deleted file mode 100644 index a80db6a7e61..00000000000 Binary files a/src/backend/dispatch-k8s-manager/manifests/chart/kubernetes-manager-0.0.34.tgz and /dev/null differ diff --git a/src/backend/dispatch-k8s-manager/manifests/chart/kubernetes-manager-0.0.36.tgz b/src/backend/dispatch-k8s-manager/manifests/chart/kubernetes-manager-0.0.36.tgz new file mode 100644 index 00000000000..0d017672836 Binary files /dev/null and b/src/backend/dispatch-k8s-manager/manifests/chart/kubernetes-manager-0.0.36.tgz differ diff --git a/src/backend/dispatch-k8s-manager/manifests/chart/templates/kubernetes-manager-configmap.yaml b/src/backend/dispatch-k8s-manager/manifests/chart/templates/kubernetes-manager-configmap.yaml index a1600e6826c..0b8c359d842 100644 --- a/src/backend/dispatch-k8s-manager/manifests/chart/templates/kubernetes-manager-configmap.yaml +++ b/src/backend/dispatch-k8s-manager/manifests/chart/templates/kubernetes-manager-configmap.yaml @@ -80,9 +80,9 @@ data: cfs: path: /data/cfs volumeMount: - dataPath: /data/landun/workspace - logPath: /data/devops/logs - builderConfigMapPath: /data/landun/config + dataPath: {{ .Values.kubernetesManager.volumeMount.dataPath }} + logPath: {{ .Values.kubernetesManager.volumeMount.logPath }} + builderConfigMapPath: /data/devops/config cfs: path: /data/bkdevops/apps readOnly: true @@ -114,4 +114,4 @@ data: {{- .Values.kubeConfig.content | nindent 4 }} {{- end -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/src/backend/dispatch-k8s-manager/manifests/chart/values.yaml b/src/backend/dispatch-k8s-manager/manifests/chart/values.yaml index a357def106f..01782a5852b 100644 --- a/src/backend/dispatch-k8s-manager/manifests/chart/values.yaml +++ b/src/backend/dispatch-k8s-manager/manifests/chart/values.yaml @@ -148,7 +148,10 @@ kubernetesManager: key: Devops-Token value: landun rsaPrivateKey: | - + volumeMount: + # 流水线构建工作空间和agent日志在容器内的挂载点 + dataPath: /data/devops/workspace + logPath: /data/devops/logs dockerInit: # 是否使用当前chart的 dockerinit.sh useDockerInit: true diff --git a/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserTurboEngineConfigController.kt b/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserTurboEngineConfigController.kt index d7043728fc4..b77292602e1 100644 --- a/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserTurboEngineConfigController.kt +++ b/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserTurboEngineConfigController.kt @@ -1,7 +1,9 @@ package com.tencent.devops.turbo.api import com.tencent.devops.api.pojo.Response +import com.tencent.devops.common.util.constants.AUTH_HEADER_DEVOPS_PROJECT_ID import com.tencent.devops.common.util.constants.AUTH_HEADER_DEVOPS_USER_ID +import com.tencent.devops.turbo.pojo.ParamEnumModel import com.tencent.devops.turbo.pojo.TurboEngineConfigModel import com.tencent.devops.turbo.validate.TurboEngineConfigGroup import com.tencent.devops.turbo.vo.TurboEngineConfigVO @@ -19,6 +21,7 @@ import org.springframework.web.bind.annotation.PutMapping import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestHeader import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam @Api(tags = ["TURBO_ENGINE_CONFIG"], description = "编译加速模式配置接口") @RequestMapping("/user/turboEngineConfig") @@ -124,4 +127,21 @@ interface IUserTurboEngineConfigController { @PathVariable("planId") planId: String ): Response + + @ApiOperation("根据区域队列名获取对应的编译器版本清单") + @GetMapping( + "/{engineCode}/compilerVersions", + produces = [MediaType.APPLICATION_JSON_VALUE] + ) + fun getCompilerVersionListByQueueName( + @ApiParam(value = "引擎标识", required = true) + @PathVariable("engineCode") + engineCode: String, + @ApiParam(value = "项目id", required = true) + @RequestHeader(AUTH_HEADER_DEVOPS_PROJECT_ID) + projectId: String, + @ApiParam(value = "队列名称", required = false) + @RequestParam("queueName") + queueName: String? + ): Response> } diff --git a/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/pojo/ParamConfigModel.kt b/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/pojo/ParamConfigModel.kt index 038ff8e9212..fb06846c2a2 100644 --- a/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/pojo/ParamConfigModel.kt +++ b/src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/pojo/ParamConfigModel.kt @@ -23,5 +23,9 @@ data class ParamConfigModel( @ApiModelProperty("是否必填") var required: Boolean? = false, @ApiModelProperty("参数tips") - var tips: String? = null + var tips: String? = null, + @ApiModelProperty("值为remote表示接口获取,其它则表示默认") + var dataType: String? = null, + @ApiModelProperty("与dataType搭配使用,表示接口地址") + var paramUrl: String? = null ) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserTurboEngineConfigController.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserTurboEngineConfigController.kt index 37b21538f92..4e87157da69 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserTurboEngineConfigController.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserTurboEngineConfigController.kt @@ -2,6 +2,7 @@ package com.tencent.devops.turbo.controller import com.tencent.devops.api.pojo.Response import com.tencent.devops.turbo.api.IUserTurboEngineConfigController +import com.tencent.devops.turbo.pojo.ParamEnumModel import com.tencent.devops.turbo.pojo.TurboEngineConfigModel import com.tencent.devops.turbo.service.TurboEngineConfigService import com.tencent.devops.turbo.service.TurboPlanService @@ -20,7 +21,9 @@ class UserTurboEngineConfigController @Autowired constructor( private val logger = LoggerFactory.getLogger(UserTurboEngineConfigController::class.java) } - override fun getEngineConfigList(projectId: String): Response> { + override fun getEngineConfigList( + projectId: String + ): Response> { return Response.success(turboEngineConfigService.getEngineConfigList(projectId)) } @@ -74,4 +77,18 @@ class UserTurboEngineConfigController @Autowired constructor( } return Response.success(turboEngineConfigService.queryEngineConfigInfo(turboPlanDetailVO.engineCode)) } + + override fun getCompilerVersionListByQueueName( + engineCode: String, + projectId: String, + queueName: String? + ): Response> { + return Response.success( + turboEngineConfigService.getCompilerVersionListByQueueName( + engineCode = engineCode, + projectId = projectId, + queueName = queueName + ) + ) + } } diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dto/ParamEnumDto.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dto/ParamEnumDto.kt new file mode 100644 index 00000000000..f5ffa7123b0 --- /dev/null +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dto/ParamEnumDto.kt @@ -0,0 +1,12 @@ +package com.tencent.devops.turbo.dto + +import com.fasterxml.jackson.annotation.JsonProperty + +data class ParamEnumDto( + @JsonProperty("param_value") + val paramValue: Any, + @JsonProperty("param_name") + val paramName: String, + @JsonProperty("visual_range") + val visualRange: List = listOf() +) diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt index fbdce248324..fb3a1cea2c2 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt @@ -8,6 +8,7 @@ import com.tencent.devops.common.util.JsonUtil import com.tencent.devops.turbo.config.TBSProperties import com.tencent.devops.turbo.dto.DistccRequestBody import com.tencent.devops.turbo.dto.DistccResponse +import com.tencent.devops.turbo.dto.ParamEnumDto import com.tencent.devops.turbo.dto.TBSTurboStatDto import com.tencent.devops.turbo.dto.WhiteListDto import com.tencent.devops.web.util.SpringContextHolder @@ -172,4 +173,22 @@ object TBSSdkApi { return responseBody } } + + /** + * 查询编译加速地区关联的编译环境 + */ + fun queryTurboCompileList(engineCode: String, queryParam: Map): List{ + val responseStr = tbsCommonRequest( + engineCode = engineCode, + resourceName = "images", + queryParam = queryParam + ) + val response: DistccResponse> = + JsonUtil.to(responseStr, object : TypeReference>>() {}) + if (response.code != 0 || !response.result) { + logger.warn("response not success!") + throw TurboException(errorCode = TURBO_THIRDPARTY_SYSTEM_FAIL, errorMessage = "fail to invoke request") + } + return response.data ?: listOf() + } } diff --git a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/TurboEngineConfigService.kt b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/TurboEngineConfigService.kt index 7910f6bc8ef..cad26501847 100644 --- a/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/TurboEngineConfigService.kt +++ b/src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/service/TurboEngineConfigService.kt @@ -19,6 +19,7 @@ import com.tencent.devops.turbo.pojo.ParamEnumSimpleModel import com.tencent.devops.turbo.pojo.TurboDisplayFieldModel import com.tencent.devops.turbo.pojo.TurboEngineConfigModel import com.tencent.devops.turbo.pojo.TurboEngineConfigPriorityModel +import com.tencent.devops.turbo.sdk.TBSSdkApi import com.tencent.devops.turbo.vo.TurboEngineConfigVO import org.quartz.CronScheduleBuilder import org.quartz.JobBuilder @@ -181,7 +182,9 @@ class TurboEngineConfigService @Autowired constructor( tips = it.tips, displayed = it.displayed, defaultValue = it.defaultValue, - required = it.required + required = it.required, + dataType = it.dataType, + paramUrl = it.paramUrl ) }, displayFields = displayFields!!.map { @@ -263,7 +266,9 @@ class TurboEngineConfigService @Autowired constructor( tips = it.tips, displayed = it.displayed, defaultValue = it.defaultValue, - required = it.required + required = it.required, + dataType = it.dataType, + paramUrl = it.paramUrl ) }, displayFields = displayFields?.map { @@ -558,7 +563,9 @@ class TurboEngineConfigService @Autowired constructor( tips = param.tips, displayed = param.displayed, defaultValue = param.defaultValue, - required = param.required + required = param.required, + dataType = param.dataType, + paramUrl = param.paramUrl ) }, recommend = it.recommend, @@ -731,4 +738,38 @@ class TurboEngineConfigService @Autowired constructor( parser.parseExpression(turboEngineConfigEntity.spelExpression) } } + + /** + * 根据区域队列名获取对应的编译器版本清单 + */ + fun getCompilerVersionListByQueueName( + engineCode: String, + projectId: String, + queueName: String? + ): List { + logger.info("request param[$engineCode | $projectId | $queueName]") + if (queueName.isNullOrEmpty()) { + logger.info("getCompilerVersionList queueName is invalid") + return listOf() + } + + val queryTurboCompileList = TBSSdkApi.queryTurboCompileList( + engineCode = engineCode, + queryParam = mapOf( + "queue_name" to queueName + ) + ) + logger.info("queryTurboCompileList: ${queryTurboCompileList.size}") + return queryTurboCompileList + .filter { + it.visualRange.isEmpty() || it.visualRange.contains(projectId) + } + .map { + ParamEnumModel( + paramValue = it.paramValue, + paramName = it.paramName, + visualRange = it.visualRange + ) + } + } } diff --git a/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/pojo/ParamConfigEntity.kt b/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/pojo/ParamConfigEntity.kt index 31146da9a70..430fc77a193 100644 --- a/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/pojo/ParamConfigEntity.kt +++ b/src/backend/turbo/model-turbo/src/main/kotlin/com/tencent/devops/turbo/model/pojo/ParamConfigEntity.kt @@ -21,5 +21,9 @@ data class ParamConfigEntity( @Field("required") var required: Boolean? = false, @Field("tips") - var tips: String? = null + var tips: String? = null, + @Field("data_type") + var dataType: String? = null, + @Field("param_url") + var paramUrl: String? = null ) diff --git a/src/frontend/bk-pipeline/package.json b/src/frontend/bk-pipeline/package.json index 4aea2b784fe..e5c8749f3a4 100644 --- a/src/frontend/bk-pipeline/package.json +++ b/src/frontend/bk-pipeline/package.json @@ -1,6 +1,6 @@ { "name": "bkui-pipeline", - "version": "0.1.28", + "version": "0.1.29", "description": "devops pipeline", "main": "dist/bk-pipeline.min.js", "scripts": { @@ -34,6 +34,7 @@ "babel-plugin-transform-vue-jsx": "^4.0.1", "copy-webpack-plugin": "9.0.1", "css-loader": "^6.4.0", + "css-minimizer-webpack-plugin": "^4.2.1", "eslint": "^7.3.1", "eslint-config-standard": "^16.0.3", "eslint-friendly-formatter": "~4.0.1", @@ -47,14 +48,16 @@ "mini-css-extract-plugin": "^2.4.2", "sass": "^1.42.1", "sass-loader": "^12.1.0", + "semver": "6.3.0", "style-loader": "^3.3.1", + "terser-webpack-plugin": "^5.3.6", "vue-loader": "^15.9.8", "vue-template-compiler": "2.6.12", "webpack": "^5.58.1" }, "dependencies": { "vue": "2.6.12", - "bk-magic-vue": "^2.4.14", + "bk-magic-vue": "2.4.15-beta.7", "vuedraggable": "^2.24.3" }, "repository": { @@ -69,4 +72,4 @@ "url": "https://github.com/Tencent/bk-ci/issues" }, "homepage": "https://github.com/Tencent/bk-ci#readme" -} +} \ No newline at end of file diff --git a/src/frontend/bk-pipeline/src/MatrixGroup.vue b/src/frontend/bk-pipeline/src/MatrixGroup.vue index 76e836dc99b..c6bb0f29e19 100644 --- a/src/frontend/bk-pipeline/src/MatrixGroup.vue +++ b/src/frontend/bk-pipeline/src/MatrixGroup.vue @@ -198,6 +198,7 @@ display: block; margin-right: 10px; transition: all 0.3s ease; + flex-shrink: 0; &.open { transform: rotate(-180deg); } diff --git a/src/frontend/bk-pipeline/src/Stage.vue b/src/frontend/bk-pipeline/src/Stage.vue index 13ac29c01c5..ba48f9968ed 100755 --- a/src/frontend/bk-pipeline/src/Stage.vue +++ b/src/frontend/bk-pipeline/src/Stage.vue @@ -369,7 +369,7 @@ } document.addEventListener('click', this.hideAddStage) }, - beforeDestroyed () { + beforeDestroy () { window.removeEventListener('click', this.hideAddStage) }, updated () { diff --git a/src/frontend/bk-pipeline/src/util.js b/src/frontend/bk-pipeline/src/util.js index e02993ab7a5..a0a5286dee6 100644 --- a/src/frontend/bk-pipeline/src/util.js +++ b/src/frontend/bk-pipeline/src/util.js @@ -133,7 +133,6 @@ export function getDependOnDesc (job) { } return val } catch (e) { - console.log(job, e) return '' } } diff --git a/src/frontend/common-lib/permission-conf.js b/src/frontend/common-lib/permission-conf.js index 1796a961d02..88934a81e58 100644 --- a/src/frontend/common-lib/permission-conf.js +++ b/src/frontend/common-lib/permission-conf.js @@ -12,27 +12,27 @@ export const resourceMap = { } export const resourceTypeMap = { - 'CODE_REPERTORY': 'repertory', - 'PIPELINE_DEFAULT': 'pipeline', - 'TICKET_CREDENTIAL': 'credential', - 'TICKET_CERT': 'cert', - 'ENVIRONMENT_ENVIRONMENT': 'environment', - 'ENVIRONMENT_ENV_NODE': 'env_node', - 'PROJECT': 'project', - 'QUALITY_RULE': 'rule', - 'QUALITY_GROUP': 'group' + CODE_REPERTORY: 'repertory', + PIPELINE_DEFAULT: 'pipeline', + TICKET_CREDENTIAL: 'credential', + TICKET_CERT: 'cert', + ENVIRONMENT_ENVIRONMENT: 'environment', + ENVIRONMENT_ENV_NODE: 'env_node', + PROJECT: 'project', + QUALITY_RULE: 'rule', + QUALITY_GROUP: 'group' } export const resourceAliasMap = { - 'CODE_REPERTORY': '代码库', - 'PIPELINE_DEFAULT': '流水线', - 'TICKET_CREDENTIAL': '凭据', - 'TICKET_CERT': '证书', - 'ENVIRONMENT_ENVIRONMENT': '环境', - 'ENVIRONMENT_ENV_NODE': '节点', - 'PROJECT': '项目', - 'QUALITY_RULE': '质量规则', - 'QUALITY_GROUP': '规则集' + CODE_REPERTORY: '代码库', + PIPELINE_DEFAULT: '流水线', + TICKET_CREDENTIAL: '凭据', + TICKET_CERT: '证书', + ENVIRONMENT_ENVIRONMENT: '环境', + ENVIRONMENT_ENV_NODE: '节点', + PROJECT: '项目', + QUALITY_RULE: '质量规则', + QUALITY_GROUP: '规则集' } export function isProjectResource (resourceId) { @@ -56,56 +56,56 @@ export const actionMap = { } export const actionAliasMap = { - 'CREATE': { - 'value': 'create', - 'alias': '创建' + CREATE: { + value: 'create', + alias: '创建' }, - 'DEPLOY': { - 'value': 'deploy', - 'alias': '部署' + DEPLOY: { + value: 'deploy', + alias: '部署' }, - 'DOWNLOAD': { - 'value': 'download', - 'alias': '下载' + DOWNLOAD: { + value: 'download', + alias: '下载' }, - 'EDIT': { - 'value': 'edit', - 'alias': '编辑' + EDIT: { + value: 'edit', + alias: '编辑' }, - 'DELETE': { - 'value': 'delete', - 'alias': '删除' + DELETE: { + value: 'delete', + alias: '删除' }, - 'VIEW': { - 'value': 'view', - 'alias': '查看' + VIEW: { + value: 'view', + alias: '查看' }, - 'MOVE': { - 'value': 'move', - 'alias': '移动' + MOVE: { + value: 'move', + alias: '移动' }, - 'USE': { - 'value': 'use', - 'alias': '使用' + USE: { + value: 'use', + alias: '使用' }, - 'SHARE': { - 'value': 'share', - 'alias': '分享' + SHARE: { + value: 'share', + alias: '分享' }, - 'LIST': { - 'value': 'list', - 'alias': '列表' + LIST: { + value: 'list', + alias: '列表' }, - 'EXECUTE': { - 'value': 'execute', - 'alias': '执行' + EXECUTE: { + value: 'execute', + alias: '执行' }, - 'ENABLE': { - 'value': 'enable', - 'alias': '停用/启用' + ENABLE: { + value: 'enable', + alias: '停用/启用' }, - 'MANAGE': { - 'value': 'manage', - 'alias': '管理' + MANAGE: { + value: 'manage', + alias: '管理' } } diff --git a/src/frontend/devops-artifactory/package.json b/src/frontend/devops-artifactory/package.json index f4c22bf1d5e..be9814e5519 100644 --- a/src/frontend/devops-artifactory/package.json +++ b/src/frontend/devops-artifactory/package.json @@ -2,6 +2,7 @@ "name": "devops-artifactory", "version": "1.0.1", "description": "", + "private": true, "dependencies": { "clipboard": "^1.7.1", "js-cookie": "^3.0.1", @@ -9,6 +10,7 @@ "vue-file-upload": "^0.1.12", "vue-property-decorator": "^6.0.0", "vue-template-compiler": "2.6.12", + "vue": "2.6.12", "vuex-class": "^0.3.0" }, "devDependencies": { @@ -33,6 +35,7 @@ "babel-loader": "^8.2.2", "babel-plugin-transform-vue-jsx": "^4.0.1", "css-loader": "^6.4.0", + "css-minimizer-webpack-plugin": "^4.2.1", "eslint": "^7.3.1", "eslint-config-standard": "^16.0.3", "eslint-friendly-formatter": "~4.0.1", @@ -47,6 +50,7 @@ "sass": "^1.42.1", "sass-loader": "^12.1.0", "style-loader": "^3.3.1", + "terser-webpack-plugin": "^5.3.6", "vue-loader": "^15.9.8", "vue-property-decorator": "^6.0.0", "vue-template-compiler": "2.6.12", @@ -65,4 +69,4 @@ }, "author": "", "gitHead": "afc92dde9341d36ec28514ede9e7cca0af3edb1b" -} \ No newline at end of file +} diff --git a/src/frontend/devops-artifactory/src/components/common/bk-trees/dialogTree.vue b/src/frontend/devops-artifactory/src/components/common/bk-trees/dialogTree.vue index fbbb1d0e549..ba00198ac62 100644 --- a/src/frontend/devops-artifactory/src/components/common/bk-trees/dialogTree.vue +++ b/src/frontend/devops-artifactory/src/components/common/bk-trees/dialogTree.vue @@ -111,7 +111,6 @@ parent = parent.$parent roadMap = `${~~parent.roadMap},${roadMap}` } - // console.log(roadMap.slice(2)) return roadMap.slice(2) }, isActive (index) { diff --git a/src/frontend/devops-atomstore/package.json b/src/frontend/devops-atomstore/package.json index 1baca96d817..e120ec40667 100755 --- a/src/frontend/devops-atomstore/package.json +++ b/src/frontend/devops-atomstore/package.json @@ -1,6 +1,7 @@ { "name": "devops-atomstore", "version": "1.0.1", + "private": true, "description": "", "dependencies": { "@blueking/bkcharts": "^2.0.10", @@ -8,6 +9,7 @@ "codemirror": "5.61.0", "core-js": "3.10.0", "mavon-editor": "^2.10.4", + "vue": "2.6.12", "moment": "^2.29.2" }, "devDependencies": { @@ -32,6 +34,7 @@ "babel-loader": "^8.2.2", "babel-plugin-transform-vue-jsx": "^4.0.1", "css-loader": "^6.4.0", + "css-minimizer-webpack-plugin": "^4.2.1", "eslint": "^7.3.1", "eslint-config-standard": "^16.0.3", "eslint-friendly-formatter": "~4.0.1", @@ -46,6 +49,7 @@ "sass": "^1.42.1", "sass-loader": "^12.1.0", "style-loader": "^3.3.1", + "terser-webpack-plugin": "^5.3.6", "vue-loader": "^15.9.8", "vue-property-decorator": "^6.0.0", "vue-template-compiler": "2.6.12", @@ -58,4 +62,4 @@ "public:master": "cross-env NODE_ENV=master webpack --mode production", "public:external": "cross-env NODE_ENV=external webpack --mode production" } -} +} \ No newline at end of file diff --git a/src/frontend/devops-atomstore/src/components/manage/detail/atom-detail/edit.vue b/src/frontend/devops-atomstore/src/components/manage/detail/atom-detail/edit.vue index ed293f83333..a045fbaefb7 100644 --- a/src/frontend/devops-atomstore/src/components/manage/detail/atom-detail/edit.vue +++ b/src/frontend/devops-atomstore/src/components/manage/detail/atom-detail/edit.vue @@ -46,7 +46,9 @@ /> - + + +