feat(apps/prod/tekton/configs): use aws-cli to upload to qiniu

apps/prod/tekton/configs/pipelines/pingcap-release-ga.yaml

@@ -12,12 +12,12 @@ spec:
     - name: oci-registry
       default: hub.pingcap.net
       description: The OCI registry stored the origin artifacts.
-    - name: qiniu-creds-secret-name
-      default: qiniu-creds
   workspaces:
     # TODO: why need to push dm offline pkgs to another bucket?
     - name: aws-secrets
       optional: true
+    - name: aws-secrets-tencent
+      optional: true
     - name: github
       description: for creating github release
   tasks:
@@ -72,12 +72,13 @@ spec:
         - "compose-offline-pkgs-amd64-community"
       taskRef:
         name: pingcap-upload-offline-package
+      workspaces:
+        - { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
       params:
         - { name: version, value: "$(params.version)" }
         - { name: edition, value: community }
         - { name: os, value: linux }
         - { name: arch, value: amd64 }
-        - { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
     - name: compose-offline-pkgs-amd64-enterprise
       runAfter: [wait-delivery-tiup]
       taskRef: { name: pingcap-compose-offline-pkgs }
@@ -98,7 +99,8 @@ spec:
         - { name: edition, value: enterprise }
         - { name: os, value: linux }
         - { name: arch, value: amd64 }
-        - { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
+      workspaces:
+        - { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
     - name: compose-offline-pkgs-amd64-dm
       runAfter: [wait-delivery-tiup]
       taskRef: { name: pingcap-compose-offline-pkgs }
@@ -116,12 +118,12 @@ spec:
         name: pingcap-upload-offline-package
       workspaces:
         - { name: aws-secrets, workspace: aws-secrets }
+        - { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
       params:
         - { name: version, value: "$(params.version)" }
         - { name: edition, value: dm }
         - { name: os, value: linux }
         - { name: arch, value: amd64 }
-        - { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
     - name: compose-offline-pkgs-arm64-community
       runAfter: [wait-delivery-tiup]
       taskRef: { name: pingcap-compose-offline-pkgs }
@@ -137,12 +139,13 @@ spec:
         - "compose-offline-pkgs-arm64-community"
       taskRef:
         name: pingcap-upload-offline-package
+      workspaces:
+        - { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
       params:
         - { name: version, value: "$(params.version)" }
         - { name: edition, value: community }
         - { name: os, value: linux }
         - { name: arch, value: arm64 }
-        - { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
     - name: compose-offline-pkgs-arm64-enterprise
       runAfter: [wait-delivery-tiup]
       taskRef: { name: pingcap-compose-offline-pkgs }
@@ -158,12 +161,13 @@ spec:
         - "compose-offline-pkgs-arm64-enterprise"
       taskRef:
         name: pingcap-upload-offline-package
+      workspaces:
+        - { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
       params:
         - { name: version, value: "$(params.version)" }
         - { name: edition, value: enterprise }
         - { name: os, value: linux }
         - { name: arch, value: arm64 }
-        - { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
     - name: compose-offline-pkgs-arm64-dm
       runAfter: [wait-delivery-tiup]
       taskRef: { name: pingcap-compose-offline-pkgs }
@@ -181,17 +185,18 @@ spec:
         name: pingcap-upload-offline-package
       workspaces:
         - { name: aws-secrets, workspace: aws-secrets }
+        - { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
       params:
         - { name: version, value: "$(params.version)" }
         - { name: edition, value: dm }
         - { name: os, value: linux }
         - { name: arch, value: arm64 }
-        - { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
     - name: upload-plugin-tarballs
       runAfter:
         - ga-tag-oci-artifacts
       taskRef:
         name: pingcap-upload-enterprise-plugins
+      workspaces:
+        - { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
       params:
         - { name: version, value: "$(params.version)" }
-        - { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }

apps/prod/tekton/configs/tasks/gen-build-binaries-scripts.yaml

@@ -26,7 +26,7 @@ spec:
     - name: git-ref
     - name: git-sha
     - name: build-image
-      default: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      default: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
     - name: save-file
       default: build-package-artifacts.sh
   steps:
@@ -48,4 +48,3 @@ spec:
           $(workspaces.output.path)/$(params.save-file)
 
         cat $(workspaces.output.path)/$(params.save-file)
-

apps/prod/tekton/configs/tasks/gen-build-images-scripts.yaml

@@ -26,7 +26,7 @@ spec:
     - name: git-ref
     - name: git-sha
     - name: build-image
-      default: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      default: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
     - name: save-file
       default: build-package-images.sh
   steps:
@@ -48,4 +48,3 @@ spec:
           $(workspaces.output.path)/$(params.save-file)
 
         cat $(workspaces.output.path)/$(params.save-file)
-

apps/prod/tekton/configs/tasks/multi-arch-image-collect.yaml

@@ -29,7 +29,7 @@ spec:
       description: tags pushed
   steps:
     - name: prepare-manifest
-      image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
       workingDir: /workspace
       script: |
         #! /usr/bin/env bash

apps/prod/tekton/configs/tasks/pingcap-auto-add-image-major-tag.yaml

@@ -5,13 +5,13 @@ metadata:
   labels:
     app.kubernetes.io/version: "0.1"
   annotations:
-    tekton.dev/platforms: "linux/amd64,linux/arm64"  
+    tekton.dev/platforms: "linux/amd64,linux/arm64"
 spec:
   params:
     - name: image_url
       description: The image full url for pull
   stepTemplate:
-    image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+    image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
   steps:
     - name: add-nightly-tag
       script: |
@@ -29,27 +29,27 @@ spec:
 
     - name: add-latest-tag
       script: |
-          pushed_repo="$(echo $(params.image_url) | cut -d ':' -f 1)"
-          pushed_tag="$(echo $(params.image_url) | cut -d ':' -f 2)"
+        pushed_repo="$(echo $(params.image_url) | cut -d ':' -f 1)"
+        pushed_tag="$(echo $(params.image_url) | cut -d ':' -f 2)"
 
-          # Check if the tag is the latest version and add "latest"
-          if [ "$(echo $pushed_tag | grep -E '^v\d+\.\d+\.+\d+$')"  ]; then
-            echo "Checking if tag is the latest version"
-            # Retrieve existing tags
-            EXISTING_TAGS=$(oras repo tags ${pushed_repo} | grep -E '^v\d+\.\d+\.+\d+$')
+        # Check if the tag is the latest version and add "latest"
+        if [ "$(echo $pushed_tag | grep -E '^v\d+\.\d+\.+\d+$')"  ]; then
+          echo "Checking if tag is the latest version"
+          # Retrieve existing tags
+          EXISTING_TAGS=$(oras repo tags ${pushed_repo} | grep -E '^v\d+\.\d+\.+\d+$')
 
-            # Determine if the tag is the latest version
-            LATEST_TAG=$(echo "$EXISTING_TAGS" | sort -V | tail -n 1)
-            if [ "$LATEST_TAG" == "$pushed_tag" ]; then
-              echo "🚀 Adding latest tag"
-              oras tag $(params.image_url) latest
-              echo "✅ Added latest tag"
-            else
-              echo "🤚 Tag is not the latest version"
-            fi
+          # Determine if the tag is the latest version
+          LATEST_TAG=$(echo "$EXISTING_TAGS" | sort -V | tail -n 1)
+          if [ "$LATEST_TAG" == "$pushed_tag" ]; then
+            echo "🚀 Adding latest tag"
+            oras tag $(params.image_url) latest
+            echo "✅ Added latest tag"
           else
-            echo "🤚 Pushed tag is not a version tag"
+            echo "🤚 Tag is not the latest version"
           fi
+        else
+          echo "🤚 Pushed tag is not a version tag"
+        fi
   workspaces:
     - description: Includes a docker `config.json`
       mountPath: /root/.docker

apps/prod/tekton/configs/tasks/pingcap-build-binaries-darwin.yaml

@@ -38,20 +38,20 @@ spec:
     - name: git-ref
     - name: git-sha
     - name: builder-image
-      default: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      default: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
     - name: release-dir
       default: build
     - name: push
       type: string
-      default: 'false'
+      default: "false"
     - name: registry
       default: hub.pingcap.net
     - name: boskos-server-url
       description: The URL of the running boskos server
       default: "http://boskos.test-pods.svc.cluster.local"
   steps:
     - name: generate-build-script
-      image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
       script: |
         git clone --depth=1 --branch=main https://github.com/PingCAP-QE/artifacts.git /workspace/artifacts
 
@@ -134,7 +134,7 @@ spec:
           --boskos.owner $(context.taskRun.name) \
           --releaseDir $(params.release-dir)
     - name: publish
-      image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
       workingDir: $(workspaces.source.path)/$(params.component)
       script: |
         script="/workspace/build-package-artifacts.sh"

apps/prod/tekton/configs/tasks/pingcap-build-binaries-linux.yaml

@@ -44,17 +44,17 @@ spec:
     - name: git-ref
     - name: git-sha
     - name: builder-image
-      default: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      default: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
     - name: release-dir
       default: build
     - name: push
       type: string
-      default: 'false'
+      default: "false"
     - name: registry
       default: hub.pingcap.net
   steps:
     - name: generate-build-script
-      image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
       script: |
         git clone --depth=1 --branch=main https://github.com/PingCAP-QE/artifacts.git /workspace/artifacts
 
@@ -88,13 +88,13 @@ spec:
       workingDir: $(workspaces.source.path)/$(params.component)
       env:
         - name: CARGO_NET_GIT_FETCH_WITH_CLI
-          value: 'true'
+          value: "true"
         # - name: RUSTUP_DIST_SERVER
         #   value: https://rsproxy.cn
         # - name: RUSTUP_UPDATE_ROOT
         #   value: https://rsproxy.cn/rustup
         - name: GOPROXY
-          value: 'http://goproxy.apps.svc,direct'
+          value: "http://goproxy.apps.svc,direct"
         - name: CARGO_HOME
           value: /workspace/.cargo
         - name: NPM_CONFIG_REGISTRY
@@ -110,7 +110,7 @@ spec:
 
         "$script" -b -a -w "$(params.release-dir)"
     - name: publish
-      image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
       workingDir: $(workspaces.source.path)/$(params.component)
       script: |
         script="/workspace/build-package-artifacts.sh"

apps/prod/tekton/configs/tasks/pingcap-build-images.yaml

@@ -38,13 +38,13 @@ spec:
     - name: release-dir
       default: build
     - name: build
-      default: 'false'
+      default: "false"
       description: Does it need to build the binaries before packing the images?
     - name: registry
       default: hub.pingcap.net
   steps:
     - name: generate
-      image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
       script: |
         git clone --depth=1 --branch=main https://github.com/PingCAP-QE/artifacts.git /workspace/artifacts
 
@@ -102,7 +102,7 @@ spec:
           "$script" -w "$(params.release-dir)" -k ${KANIKO_EXECUTOR} -o $(results.pushed.path)
         fi
     - name: add-more-tags
-      image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
       script: |
         script="/workspace/build-package-images.sh"
         if [ ! -f "$script" ]; then

apps/prod/tekton/configs/tasks/pingcap-compose-offline-pkgs.yaml

@@ -28,14 +28,14 @@ spec:
         supports: 'community' or 'enterprise'.
     - name: push
       type: string
-      default: 'false'
+      default: "false"
     - name: registry
       default: hub.pingcap.net
     - name: tiup-mirror
       description: Product is http://tiup.pingcap.net:8987, Staging is http://tiup.pingcap.net:8988,
       default: http://tiup.pingcap.net:8988
   stepTemplate:
-    image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+    image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
     env:
       - name: OUT_SCRIPT
         value: /workspace/compose-offline-packages-artifacts.sh

apps/prod/tekton/configs/tasks/pingcap-get-builder-image.yaml

@@ -36,7 +36,7 @@ spec:
       default: ""
   steps:
     - name: get
-      image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
+      image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
       script: |
         if [ -n "$(params.force-builder-image)" ]; then
           echo "use force builder image: $(params.force-builder-image)"