ci(benchmark): run s3 clickbench with cloud

.github/actions/benchmark_cloud/action.yml

@@ -0,0 +1,105 @@
+name: "Benchmark Cloud"
+description: "Run benchmark for S3 with cloud storage"
+inputs:
+  sha:
+    description: "commit sha"
+    required: true
+  run_id:
+    description: "benchmark run id"
+    required: true
+  dataset:
+    description: "hits/tpch"
+    required: true
+  source:
+    description: "pr/release"
+    required: true
+  source_id:
+    description: "pr_id/release_tag"
+    required: true
+  size:
+    description: "Small/Medium/Large"
+    required: true
+  image_tag:
+    description: "Databend image tag"
+    required: true
+  cloud_email:
+    description: "Benchmark cloud email"
+    required: true
+  cloud_password:
+    description: "Benchmark cloud password"
+    required: true
+  cloud_org:
+    description: "Benchmark cloud org"
+    required: true
+  cloud_endpoint:
+    description: "Benchmark cloud endpoint"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: ./.github/actions/setup_bendsql
+
+    - name: Install script dependencies
+      shell: bash
+      run: |
+        sudo apt-get update -y
+        sudo apt-get install -y jq bc
+
+    - name: Run Benchmark
+      working-directory: benchmark/clickbench
+      env:
+        BENCHMARK_ID: ${{ inputs.run_id }}
+        BENCHMARK_DATASET: ${{ inputs.dataset }}
+        BENCHMARK_SIZE: ${{ inputs.size }}
+        BENCHMARK_IMAGE_TAG: ${{ inputs.image_tag }}
+        CLOUD_EMAIL: ${{ inputs.cloud_email }}
+        CLOUD_PASSWORD: ${{ inputs.cloud_password }}
+        CLOUD_ORG: ${{ inputs.cloud_org }}
+        CLOUD_ENDPOINT: ${{ inputs.cloud_endpoint }}
+      shell: bash
+      run: |
+        ./benchmark_cloud.sh
+
+    - name: Prepare Metadata
+      working-directory: benchmark/clickbench
+      shell: bash
+      run: |
+        jq ".date = \"$(date -u +%Y-%m-%d)\"" <result.json >result.json.tmp && mv result.json.tmp result.json
+        case ${{ inputs.source }} in
+          pr)
+            BENCHMARK_SYSTEM="Databend(PR#${{ inputs.source_id }})"
+            BENCHMARK_TYPE="PR"
+            ;;
+          release)
+            BENCHMARK_SYSTEM="Databend(Release@${{ inputs.source_id }})"
+            BENCHMARK_TYPE="Release"
+            ;;
+          *)
+            echo "Unspported benchmark source: ${{ inputs.source }}"
+            exit 1
+        esac
+        jq ".system = \"${BENCHMARK_SYSTEM}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
+
+        case ${{ inputs.size }} in
+          Medium)
+            BENCHMARK_CLUSTER_SIZE=16
+            ;;
+          Large)
+            BENCHMARK_CLUSTER_SIZE=64
+            ;;
+          *)
+            echo "Unspported benchmark size: ${{ inputs.size }}"
+            exit 1
+        esac
+        jq ".cluster_size = \"${BENCHMARK_CLUSTER_SIZE}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
+        jq ".machine = \"${{ inputs.size }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
+        jq ".tags = [\"${BENCHMARK_TYPE}\", \"serverless\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
+        jq ".comment = \"commit:${{ inputs.sha }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
+        mv result.json result-${{ inputs.dataset }}-cloud.json
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: benchmark-${{ inputs.dataset }}-cloud-${{ inputs.size }}
+        path: benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json

.github/actions/benchmark/action.yml → .github/actions/benchmark_local/action.yml

@@ -1,8 +1,11 @@
-name: "Benchmark"
-description: "Run benchmark"
+name: "Benchmark Local"
+description: "Run benchmark for local filesystem"
 inputs:
-  storage:
-    description: "s3/fs"
+  sha:
+    description: "commit sha"
+    required: true
+  run_id:
+    description: "benchmark run id"
     required: true
   dataset:
     description: "hits/tpch"
@@ -20,7 +23,7 @@ runs:
       uses: ./.github/actions/artifact_download
       with:
         profile: release
-        sha: ${{ github.sha }}
+        sha: ${{ inputs.sha }}
         target: x86_64-unknown-linux-gnu
 
     - name: Setup Databend Binary
@@ -31,14 +34,7 @@ runs:
         databend-query --version
         databend-meta --version
 
-    - name: Setup BendSQL
-      shell: bash
-      run: |
-        curl -fsSLo ./bendsql.tar.gz https://github.com/databendcloud/bendsql/releases/download/v0.3.5/bendsql-linux-amd64.tar.gz
-        tar -xvzf ./bendsql.tar.gz
-        chmod +x ./linux-amd64/bendsql
-        sudo cp ./linux-amd64/bendsql /usr/local/bin/bendsql
-        bendsql version
+    - uses: ./.github/actions/setup_bendsql
 
     - name: Install script dependencies
       shell: bash
@@ -49,29 +45,16 @@ runs:
     - name: Run Benchmark
       working-directory: benchmark/clickbench
       env:
-        BENCHMARK_ID: ${{ github.run_id }}
-        BENCHMARK_STORAGE: ${{ inputs.storage }}
+        BENCHMARK_ID: ${{ inputs.run_id }}
         BENCHMARK_DATASET: ${{ inputs.dataset }}
       shell: bash
       run: |
-        ./benchmark.sh
+        ./benchmark_local.sh
 
     - name: Prepare Metadata
       working-directory: benchmark/clickbench
       shell: bash
       run: |
-        case ${{ inputs.storage }} in
-          s3)
-            BENCHMARK_STORAGE="s3"
-            ;;
-          fs)
-            BENCHMARK_STORAGE="gp3"
-            ;;
-          *)
-            echo "Unspported benchmark storage: ${{ inputs.storage }}"
-            exit 1
-        esac
-
         case ${{ inputs.source }} in
           pr)
             BENCHMARK_SYSTEM="Databend(PR#${{ inputs.source_id }})"
@@ -87,18 +70,18 @@ runs:
         esac
         jq ".system = \"${BENCHMARK_SYSTEM}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
         jq ".date = \"$(date -u +%Y-%m-%d)\"" <result.json >result.json.tmp && mv result.json.tmp result.json
-        jq ".machine = \"c5.4xlarge,${BENCHMARK_STORAGE}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
+        jq ".machine = \"c5.4xlarge,gp3\"" <result.json >result.json.tmp && mv result.json.tmp result.json
         jq ".cluster_size = 1" <result.json >result.json.tmp && mv result.json.tmp result.json
-        jq ".comment = \"commit:${{ github.sha }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
-        jq ".tags = [\"${BENCHMARK_TYPE}\", \"${BENCHMARK_STORAGE}\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
+        jq ".comment = \"commit:${{ inputs.sha }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
+        jq ".tags = [\"${BENCHMARK_TYPE}\", \"gp3\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
         jq ".extra.${{ inputs.source }} = \"${{ inputs.source_id }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
-        mv result.json result-${{ inputs.dataset }}-${{ inputs.storage }}.json
+        mv result.json result-${{ inputs.dataset }}-local.json
 
     - name: Upload artifact
       uses: actions/upload-artifact@v3
       with:
-        name: benchmark-${{ inputs.dataset }}-${{ inputs.storage }}
-        path: benchmark/clickbench/result-${{ inputs.dataset }}-${{ inputs.storage }}.json
+        name: benchmark-${{ inputs.dataset }}-local
+        path: benchmark/clickbench/result-${{ inputs.dataset }}-local.json
 
     - name: Clean test data
       working-directory: benchmark/clickbench

.github/actions/setup_bendsql/action.yml

@@ -0,0 +1,18 @@
+name: "Setup BendSQL"
+description: "Setup BendSQL for CI"
+inputs:
+  version:
+    description: "bendsql version"
+    required: false
+    default: v0.3.5
+runs:
+  using: "composite"
+  steps:
+    - name: Download and Install
+      shell: bash
+      run: |
+        curl -fsSLo ./bendsql.tar.gz https://github.com/databendcloud/bendsql/releases/download/${{ inputs.version }}/bendsql-linux-amd64.tar.gz
+        tar -xvzf ./bendsql.tar.gz
+        chmod +x ./linux-amd64/bendsql
+        sudo cp ./linux-amd64/bendsql /usr/local/bin/bendsql
+        bendsql version

.github/workflows/benchmark.yml

@@ -47,21 +47,6 @@ jobs:
         with:
           target: x86_64-unknown-linux-gnu
           profile: release
-
-  clickbench:
-    needs: build_release
-    timeout-minutes: 30
-    runs-on: [self-hosted, X64, Linux, perf]
-    strategy:
-      matrix:
-        storage:
-          - s3
-          - fs
-        dataset:
-          - hits
-          - tpch
-    steps:
-      - uses: actions/checkout@v3
       - name: Get benchmark source
         run: |
           case ${{ github.event_name }} in
@@ -78,9 +63,15 @@ jobs:
               exit 1
               ;;
           esac
-      - uses: ./.github/actions/benchmark
+      - name: Generate benchmark metadata
+        run: |
+          echo '{}' > metadata.json
+          jq ".sha = \"${{ github.sha }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
+          jq ".run_id = \"${{ github.run_id }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
+          jq ".source = \"${{ env.BENCHMARK_SOURCE }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
+          jq ".source_id = \"${{ env.BENCHMARK_SOURCE_ID }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
         with:
-          storage: ${{ matrix.storage }}
-          dataset: ${{ matrix.dataset }}
-          source: ${{ env.BENCHMARK_SOURCE }}
-          source_id: ${{ env.BENCHMARK_SOURCE_ID }}
+          name: benchmark-metadata
+          path: metadata.json

.github/workflows/release.yml

@@ -343,7 +343,6 @@ jobs:
           platforms: linux/amd64,linux/arm64
           context: .
           file: ./docker/Dockerfile
-          build-args: VERSION=${{ needs.create_release.outputs.version }}
       - name: Update repo description
         uses: peter-evans/dockerhub-description@v3
         with:
@@ -416,7 +415,6 @@ jobs:
           platforms: linux/amd64,linux/arm64
           context: .
           file: ./docker/${{ matrix.distro }}/${{ matrix.service }}.Dockerfile
-          build-args: VERSION=${{ needs.create_release.outputs.version }}
 
   release_packages:
     name: release packages