Skip to content

Commit

Permalink
ci: change benchmark machine name for cloud (#11327)
Browse files Browse the repository at this point in the history
  • Loading branch information
@everpcpc
everpcpc authored May 6, 2023
1 parent 3c2622f commit 98c267e
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 170 deletions.
48 changes: 0 additions & 48 deletions .github/workflows/benchmark.yml
View file

This file was deleted.

180 changes: 60 additions & 120 deletions .github/workflows/trusted-benchmark.yml
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,36 @@
name: Benchmark (trusted)

on:
workflow_run:
workflows: ["Benchmark"]
pull_request_target:
types:
- completed
- labeled
branches:
- main
workflow_call:
inputs:
tag:
description: The tag to run benchmark
required: true
type: string

# concurrency:
# group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
# cancel-in-progress: true

permissions:
id-token: write
pull-requests: write
checks: write
statuses: write
contents: read

env:
BENCHMARK_S3_PREFIX: s3://repo.databend.rs/benchmark/clickbench
BUILD_PROFILE: release
RUNNER_PROVIDER: aws

jobs:
metadata:
if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
runs-on: [self-hosted, X64, Linux, 4c8g]
build:
if: github.event_name != 'pull_request_target' || contains(github.event.pull_request.labels.*.name, 'ci-benchmark')
runs-on: [self-hosted, X64, Linux, 16c32g]
outputs:
sha: ${{ steps.metadata.outputs.sha }}
head_sha: ${{ steps.metadata.outputs.head_sha }}
Expand All @@ -34,64 +39,35 @@ jobs:
source_id: ${{ steps.metadata.outputs.source_id }}
steps:
- uses: actions/checkout@v3
- name: Download benchmark metadata
with:
fetch-depth: 0
- name: Build Release
uses: ./.github/actions/build_linux
if: github.event_name == 'pull_request_target'
timeout-minutes: 30
with:
target: x86_64-unknown-linux-gnu
artifacts: meta,query
- name: Set Benchmark metadata
id: metadata
env:
GH_TOKEN: ${{ github.token }}
run: |
echo "sha=${{ github.sha }}" >> $GITHUB_OUTPUT
echo "run_id=${{ github.run_id }}" >> $GITHUB_OUTPUT
case ${{ github.event_name }} in
# pull request
workflow_run)
gh run download ${{ github.event.workflow_run.id }} --name benchmark-metadata
echo "sha=$(jq -r '.sha' metadata.json)" >> $GITHUB_OUTPUT
echo "head_sha=$(jq -r '.head_sha' metadata.json)" >> $GITHUB_OUTPUT
echo "run_id=$(jq -r '.run_id' metadata.json)" >> $GITHUB_OUTPUT
pull_request_target)
echo "head_sha=${{ github.event.pull_request.head.sha }}" >> $GITHUB_OUTPUT
echo "source=pr" >> $GITHUB_OUTPUT
echo "source_id=$(jq -r '.pr_id' metadata.json)" >> $GITHUB_OUTPUT
echo "source_id=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
;;
*)
echo "sha=${{ github.sha }}" >> $GITHUB_OUTPUT
echo "head_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
echo "run_id=${{ github.run_id }}" >> $GITHUB_OUTPUT
echo "source=release" >> $GITHUB_OUTPUT
echo "source_id=${{ inputs.tag }}" >> $GITHUB_OUTPUT
;;
esac
- uses: ./.github/actions/create_check_status
if: steps.metadata.outputs.source == 'pr'
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
sha: ${{ steps.metadata.outputs.head_sha }}
context: "Benchmark Local / hits"
state: pending
description: "Queued"
- uses: ./.github/actions/create_check_status
if: steps.metadata.outputs.source == 'pr'
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
sha: ${{ steps.metadata.outputs.head_sha }}
context: "Benchmark Cloud / hits (Medium)"
state: pending
description: "Queued"
- uses: ./.github/actions/create_check_status
if: steps.metadata.outputs.source == 'pr'
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
sha: ${{ steps.metadata.outputs.head_sha }}
context: "Benchmark Cloud / hits (Large)"
state: pending
description: "Queued"
- uses: ./.github/actions/create_check_status
if: steps.metadata.outputs.source == 'pr'
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
sha: ${{ steps.metadata.outputs.head_sha }}
context: "Benchmark Cloud / tpch (Large)"
state: pending
description: "Queued"
local:
needs: metadata
needs: build
timeout-minutes: 60
runs-on: [self-hosted, X64, Linux, 16c32g]
strategy:
Expand All @@ -107,19 +83,19 @@ jobs:
- name: Download artifact
run: |
mkdir -p ./target/release
case ${{ needs.metadata.outputs.source }} in
case ${{ needs.build.outputs.source }} in
pr)
aws s3 sync s3://databend-ci/release/${{ needs.metadata.outputs.sha }}/x86_64-unknown-linux-gnu/default/ \
aws s3 sync s3://databend-ci/release/${{ needs.build.outputs.sha }}/x86_64-unknown-linux-gnu/default/ \
target/release/ --exclude "*" --include "databend-*" --no-progress
;;
release)
version=${{ needs.metadata.outputs.source_id }}
version=${{ needs.build.outputs.source_id }}
target=x86_64-unknown-linux-gnu
aws s3 cp s3://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz distro/ --no-progress
tar x -C ./target/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
;;
*)
echo "Unknown source: ${{ needs.metadata.outputs.source }}"
echo "Unknown source: ${{ needs.build.outputs.source }}"
exit 1
;;
esac
Expand All @@ -130,36 +106,18 @@ jobs:
sudo cp ./target/release/databend-* /usr/local/bin/
databend-query --version
databend-meta --version
- name: Set github check status to pending
if: needs.metadata.outputs.source == 'pr'
uses: ./.github/actions/create_check_status
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
sha: ${{ needs.metadata.outputs.head_sha }}
context: "Benchmark Local / ${{ matrix.dataset }}"
state: pending
description: "Running"
- uses: ./.github/actions/benchmark_local
timeout-minutes: 30
id: benchmark
with:
sha: ${{ needs.metadata.outputs.sha }}
run_id: ${{ needs.metadata.outputs.run_id }}
sha: ${{ needs.build.outputs.sha }}
run_id: ${{ needs.build.outputs.run_id }}
dataset: ${{ matrix.dataset }}
source: ${{ needs.metadata.outputs.source }}
source_id: ${{ needs.metadata.outputs.source_id }}
- name: Set github check status to completed
if: needs.metadata.outputs.source == 'pr'
uses: ./.github/actions/create_check_status
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
sha: ${{ needs.metadata.outputs.head_sha }}
context: "Benchmark Local / ${{ matrix.dataset }}"
state: success
description: "Completed"
source: ${{ needs.build.outputs.source }}
source_id: ${{ needs.build.outputs.source_id }}

docker:
needs: metadata
needs: build
timeout-minutes: 10
runs-on: [self-hosted, X64, Linux, 4c8g]
outputs:
Expand All @@ -169,19 +127,19 @@ jobs:
- name: Download artifact
run: |
mkdir -p ./target/release
case ${{ needs.metadata.outputs.source }} in
case ${{ needs.build.outputs.source }} in
pr)
aws s3 sync s3://databend-ci/release/${{ needs.metadata.outputs.sha }}/x86_64-unknown-linux-gnu/default/ \
aws s3 sync s3://databend-ci/release/${{ needs.build.outputs.sha }}/x86_64-unknown-linux-gnu/default/ \
target/release/ --exclude "*" --include "databend-*" --no-progress
;;
release)
version=${{ needs.metadata.outputs.source_id }}
version=${{ needs.build.outputs.source_id }}
target=x86_64-unknown-linux-gnu
aws s3 cp s3://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz distro/ --no-progress
tar x -C ./target/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
;;
*)
echo "Unknown source: ${{ needs.metadata.outputs.source }}"
echo "Unknown source: ${{ needs.build.outputs.source }}"
exit 1
;;
esac
Expand All @@ -200,7 +158,7 @@ jobs:
run: |
mkdir -p ./distro/linux/amd64
cp ./target/release/databend-query ./distro/linux/amd64/databend-query
echo "tag=benchmark-${{ needs.metadata.outputs.sha }}" >> $GITHUB_OUTPUT
echo "tag=benchmark-${{ needs.build.outputs.sha }}" >> $GITHUB_OUTPUT
- name: push service image
uses: docker/build-push-action@v3
with:
Expand All @@ -211,7 +169,7 @@ jobs:
file: ./docker/debian/query.Dockerfile

cloud:
needs: [metadata, docker]
needs: [build, docker]
timeout-minutes: 90
runs-on: [self-hosted, X64, Linux, 4c8g]
strategy:
Expand All @@ -225,44 +183,26 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup_bendsql
- name: Set github check status to pending
if: needs.metadata.outputs.source == 'pr'
uses: ./.github/actions/create_check_status
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
sha: ${{ needs.metadata.outputs.head_sha }}
context: "Benchmark Cloud / ${{ matrix.dataset }} (${{ matrix.size }})"
state: pending
description: "Running"
- uses: ./.github/actions/benchmark_cloud
timeout-minutes: 60
id: benchmark
with:
sha: ${{ needs.metadata.outputs.sha }}
run_id: ${{ needs.metadata.outputs.run_id }}
sha: ${{ needs.build.outputs.sha }}
run_id: ${{ needs.build.outputs.run_id }}
dataset: ${{ matrix.dataset }}
source: ${{ needs.metadata.outputs.source }}
source_id: ${{ needs.metadata.outputs.source_id }}
source: ${{ needs.build.outputs.source }}
source_id: ${{ needs.build.outputs.source_id }}
size: ${{ matrix.size }}
version: ${{ needs.docker.outputs.tag }}
cloud_user: ${{ secrets.BENCHMARK_CLOUD_USER }}
cloud_password: ${{ secrets.BENCHMARK_CLOUD_PASSWORD }}
cloud_gateway: ${{ secrets.BENCHMARK_CLOUD_GATEWAY }}
- name: Set github check status to pending
if: needs.metadata.outputs.source == 'pr'
uses: ./.github/actions/create_check_status
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
sha: ${{ needs.metadata.outputs.head_sha }}
context: "Benchmark Cloud / ${{ matrix.dataset }} (${{ matrix.size }})"
state: success
description: "Completed"

comment_on_pr:
name: comment
runs-on: [self-hosted, X64, Linux, 4c8g]
needs: [metadata, local, cloud]
if: needs.metadata.outputs.source == 'pr'
needs: [build, local, cloud]
if: needs.build.outputs.source == 'pr'
steps:
- uses: actions/checkout@v3
- name: Install Dependencies
Expand All @@ -285,7 +225,7 @@ jobs:
path: benchmark/clickbench/results/tpch
- name: Get Report Prefix
run: |
echo "REPORT_S3_PREFIX=${{ env.BENCHMARK_S3_PREFIX }}/pr/${{ needs.metadata.outputs.source_id }}/${{ needs.metadata.outputs.run_id }}" >> $GITHUB_ENV
echo "REPORT_S3_PREFIX=${{ env.BENCHMARK_S3_PREFIX }}/pr/${{ needs.build.outputs.source_id }}/${{ needs.build.outputs.run_id }}" >> $GITHUB_ENV
- name: Upload PR clickbench result to repo.databend.rs
working-directory: benchmark/clickbench
run: |
Expand All @@ -299,8 +239,8 @@ jobs:
- name: Generate clickbench report
working-directory: benchmark/clickbench
run: |
./update_results.py --dataset hits --pr ${{ needs.metadata.outputs.source_id }}
./update_results.py --dataset tpch --pr ${{ needs.metadata.outputs.source_id }}
./update_results.py --dataset hits --pr ${{ needs.build.outputs.source_id }}
./update_results.py --dataset tpch --pr ${{ needs.build.outputs.source_id }}
- name: Upload PR clickbench report to repo.databend.rs
working-directory: benchmark/clickbench
run: |
Expand All @@ -309,19 +249,19 @@ jobs:
- name: Comment on PR
uses: everpcpc/comment-on-pr-action@v1
with:
number: ${{ needs.metadata.outputs.source_id }}
number: ${{ needs.build.outputs.source_id }}
token: ${{ github.token }}
body: |
![local](https://img.shields.io/static/v1?label=AWS%20EC2&message=c5.4xlarge&color=orange&logo=amazonec2) ![cloud](https://img.shields.io/static/v1?label=Databend%20Cloud&message=Medium&color=blue&logo=icloud) ![cloud](https://img.shields.io/static/v1?label=Databend%20Cloud&message=Large&color=blue&logo=icloud)
## ClickBench Report
* **hits**: https://repo.databend.rs/benchmark/clickbench/pr/${{ needs.metadata.outputs.source_id }}/${{ needs.metadata.outputs.run_id }}/hits.html
* **tpch**: https://repo.databend.rs/benchmark/clickbench/pr/${{ needs.metadata.outputs.source_id }}/${{ needs.metadata.outputs.run_id }}/tpch.html
* **hits**: https://repo.databend.rs/benchmark/clickbench/pr/${{ needs.build.outputs.source_id }}/${{ needs.build.outputs.run_id }}/hits.html
* **tpch**: https://repo.databend.rs/benchmark/clickbench/pr/${{ needs.build.outputs.source_id }}/${{ needs.build.outputs.run_id }}/tpch.html
archive_for_release:
name: archive
runs-on: [self-hosted, X64, Linux, 4c8g]
needs: [metadata, local, cloud]
if: needs.metadata.outputs.source == 'release'
needs: [build, local, cloud]
if: needs.build.outputs.source == 'release'
strategy:
matrix:
dataset:
Expand All @@ -347,11 +287,11 @@ jobs:
working-directory: benchmark/clickbench
run: |
aws s3 sync ${{ env.BENCHMARK_S3_PREFIX }}/release/${{ matrix.dataset }}/$(date -u +%Y)/$(date -u +%m)/ ./results/${{ matrix.dataset }}/ --no-progress
./update_results.py --dataset ${{ matrix.dataset }} --release ${{ needs.metadata.outputs.source_id }}
./update_results.py --dataset ${{ matrix.dataset }} --release ${{ needs.build.outputs.source_id }}
- name: Upload nightly results to repo.databend.rs
working-directory: benchmark/clickbench/
run: |
RESULT_PREFIX="${{ env.BENCHMARK_S3_PREFIX }}/release/${{ matrix.dataset }}/$(date -u +%Y)/$(date -u +%m)/$(date -u +%Y-%m-%d)/${{ needs.metadata.outputs.source_id }}"
RESULT_PREFIX="${{ env.BENCHMARK_S3_PREFIX }}/release/${{ matrix.dataset }}/$(date -u +%Y)/$(date -u +%m)/$(date -u +%Y-%m-%d)/${{ needs.build.outputs.source_id }}"
LATEST_PREFIX="${{ env.BENCHMARK_S3_PREFIX }}/release/${{ matrix.dataset }}/latest/latest"
for file in ./results/${{ matrix.dataset }}/*.json; do
aws s3 cp $file "${RESULT_PREFIX}-$(basename $file)"
Expand Down
Loading

1 comment on commit 98c267e

@vercel
Copy link

@vercel vercel bot commented on 98c267e May 6, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

databend – ./

databend.vercel.app
databend-databend.vercel.app
databend-git-main-databend.vercel.app
databend.rs

Please sign in to comment.