From e144ce31e8d71b5e6400e08293f9b362be9b4564 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Fri, 8 Jul 2022 20:02:43 -0700 Subject: [PATCH 01/13] Adds a github action to support x64 performance testing using a sightglass This github action allows performance testing using sightglass. The action is triggered either via a workflow dispatch or with the comment '/bench_x64', in a pull request. Once triggered the action will send a request to a private repository that supports using a self-hosted runner to do comparisons of "refs/feature/commit" vs "refs/heads/main" for wasmtime. If the action is triggered via a comment in a pull request (with '/bench_x64') then the commit referenced by the pull request is used for the comparison against refs/head/main. If triggered via a workflow dispatch the interface will request the commit to compare against refs/head/main. The results of the performance tests, run via sightglass, will be a table showing a percentage change in clock ticks in various stages requried for executing the benchmark, namely instantiate, compiliation, and execution. This patch is intended to be just a starting patch with much to tweak and improve. One of the TODOs will be adding support for aarch64 .. currently this patch supports only x64. Note also that the logic for actually doing the comparison and parsing the results occurs with the action associated with the private repo and so this patch itself (though the trigger) is fairly straight forward. --- .github/workflows/performance.yml | 98 +++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 .github/workflows/performance.yml diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 000000000000..65024e9907e5 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,98 @@ +# This is a workflow triggered by PR or triggered manually +# Runs quick performance tests and reports the comparison against HEAD +# Test should take less than 10 minutes to run on current self-hosted devices +name: "Performance Testing" + +# Controls when the action will run. +# This workflow runs when manually triggered using the UI or API. +on: + issue_comment: + types: [created, edited] + pull_request_review_comment: + types: [created, edited] + workflow_dispatch: + inputs: + actor: + description: 'User that requested the workflow' + type: string + required: false + pr_number: + description: 'Pull request number for where to post the results' + type: string + required: false + default: '' + post_results: + description: 'Check to post results to the given PR' + type: boolean + required: false + default: false + repository: + description: 'Repository for the commit' + type: string + default: 'bytecodealliance/wasmtime' + required: true + refs: + description: 'Ref for the commit' + type: string + default: 'refs/heads/main' + required: true + sg_commit: + description: 'Sightglass ref' + type: string + default: '81c425a' + required: false + message: + description: 'Message to print with posted PR comment' + type: string + default: 'Triggered by wasmtime workflow dispatch' + required: false + +# Env variables +env: + SG_COMMIT: 649509c + TOKEN: ${{ secrets.SIGHTGLASS_BENCHMARKING_TOKEN }} + GITHUB_CONTEXT: ${{ toJson(github) }} + +jobs: + Sightglass_x86-64_from_pr_comment: + name: Sightglass x86-64_from_pr_comment + runs-on: ubuntu-latest + if: | + startsWith(github.event.comment.body, '/bench_x64') && + (github.event.issue.pull_request) && + (('abrown' == github.event.comment.user.login) + || ('afonso360' == github.event.comment.user.login) + || ('akirilov-arm' == github.event.comment.user.login) + || ('alexcrichton' == github.event.comment.user.login) + || ('bbouvier' == github.event.comment.user.login) + || ('bjorn3' == github.event.comment.user.login) + || ('cfallin' == github.event.comment.user.login) + || ('fitzgen' == github.event.comment.user.login) + || ('jlb6740' == github.event.comment.user.login) + || ('sparker-arm' == github.event.comment.user.login) + || ('uweigand' == github.event.comment.user.login)) + steps: + - run: echo "MESSAGE=Requested from pull request comment." >> $GITHUB_ENV + - run: echo "ACTOR=${{ github.event.comment.user.login }}" >> $GITHUB_ENV + - run: echo "REPOSITORY=${{ github.repository }}" >> $GITHUB_ENV + - run: echo "REFS=${{ github.ref }}" >> $GITHUB_ENV + - run: echo "PR_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV + - run: echo "PUBLISH=true" >> $GITHUB_ENV + - run: echo "$GITHUB_CONTEXT" + - run: | + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ env.TOKEN }}" https://api.github.com/repos/bytecodealliance/wasmtime-sightglass-benchmarking/dispatches -d '{"event_type":"bench-sightglass", "client_payload":{"message":"${{ env.MESSAGE }}", "actor":"${{ env.ACTOR }}", "repository":"${{ env.REPOSITORY }}", "ref":"${{ env.REFS }}", "pr_number":"${{ env.PR_NUMBER }}", "publish":"${{ env.PUBLISH }}" }' + + Sightglass_x86-64_from_workflow_dispatch: + name: Performance x86-64_from_workflow_dispatch + runs-on: ubuntu-latest + if: (github.event_name == 'workflow_dispatch') + steps: + - run: echo "MESSAGE=${{ github.event.inputs.message }}" >> $GITHUB_ENV + - run: echo "ACTOR=${{ github.event.inputs.actor }}" >> $GITHUB_ENV + - run: echo "REPOSITORY=${{ github.event.inputs.repository }}" >> $GITHUB_ENV + - run: echo "REFS=${{ github.event.inputs.refs }}" >> $GITHUB_ENV + - run: echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV + - run: echo "PUBLISH=${{ github.event.inputs.post_results }}" >> $GITHUB_ENV + - run: echo "$GITHUB_CONTEXT" + - run: | + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ env.TOKEN }}" https://api.github.com/repos/bytecodealliance/wasmtime-sightglass-benchmarking/dispatches -d '{"event_type":"bench-sightglass", "client_payload":{"message":"${{ env.MESSAGE }}", "actor":"${{ env.ACTOR }}", "repository":"${{ env.REPOSITORY }}", "ref":"${{ env.REFS }}", "pr_number":"${{ env.PR_NUMBER }}", "publish":"${{ env.PUBLISH }}" }' \ No newline at end of file From a66b14f5a741a21ce49a6a5654ffc7a677fb554a Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Wed, 14 Sep 2022 02:06:20 -0700 Subject: [PATCH 02/13] Refactor patch to consolidate all steps to here. --- .github/workflows/performance.yml | 211 ++++++++++++++++++++---------- 1 file changed, 143 insertions(+), 68 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 65024e9907e5..c6b183e3f1dc 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -6,93 +6,168 @@ name: "Performance Testing" # Controls when the action will run. # This workflow runs when manually triggered using the UI or API. on: - issue_comment: - types: [created, edited] + pull_request_review: + types: [submitted, edited] pull_request_review_comment: types: [created, edited] - workflow_dispatch: - inputs: - actor: - description: 'User that requested the workflow' - type: string - required: false - pr_number: - description: 'Pull request number for where to post the results' - type: string - required: false - default: '' - post_results: - description: 'Check to post results to the given PR' - type: boolean - required: false - default: false - repository: - description: 'Repository for the commit' - type: string - default: 'bytecodealliance/wasmtime' - required: true - refs: - description: 'Ref for the commit' - type: string - default: 'refs/heads/main' - required: true - sg_commit: - description: 'Sightglass ref' - type: string - default: '81c425a' - required: false - message: - description: 'Message to print with posted PR comment' - type: string - default: 'Triggered by wasmtime workflow dispatch' - required: false + push: # Env variables env: - SG_COMMIT: 649509c + SG_COMMIT: b4971ae TOKEN: ${{ secrets.SIGHTGLASS_BENCHMARKING_TOKEN }} GITHUB_CONTEXT: ${{ toJson(github) }} jobs: - Sightglass_x86-64_from_pr_comment: - name: Sightglass x86-64_from_pr_comment + Wasmtime_Repo_On_PR_Comment: + name: Benchmark x64 on PR comment Wasmtime repo runs-on: ubuntu-latest if: | - startsWith(github.event.comment.body, '/bench_x64') && - (github.event.issue.pull_request) && - (('abrown' == github.event.comment.user.login) - || ('afonso360' == github.event.comment.user.login) - || ('akirilov-arm' == github.event.comment.user.login) - || ('alexcrichton' == github.event.comment.user.login) - || ('bbouvier' == github.event.comment.user.login) - || ('bjorn3' == github.event.comment.user.login) - || ('cfallin' == github.event.comment.user.login) - || ('fitzgen' == github.event.comment.user.login) - || ('jlb6740' == github.event.comment.user.login) - || ('sparker-arm' == github.event.comment.user.login) - || ('uweigand' == github.event.comment.user.login)) + (github.event_name == 'pull_request_review') && + (startsWith(github.event.review.body, '/bench_x64')) && + (('abrown' == github.event.review.user.login) + || ('afonso360' == github.event.review.user.login) + || ('akirilov-arm' == github.event.review.user.login) + || ('alexcrichton' == github.event.review.user.login) + || ('bbouvier' == github.event.review.user.login) + || ('bjorn3' == github.event.review.user.login) + || ('cfallin' == github.event.review.user.login) + || ('fitzgen' == github.event.review.user.login) + || ('jlb6740' == github.event.review.user.login) + || ('sparker-arm' == github.event.review.user.login) + || ('uweigand' == github.event.review.user.login)) steps: - run: echo "MESSAGE=Requested from pull request comment." >> $GITHUB_ENV - - run: echo "ACTOR=${{ github.event.comment.user.login }}" >> $GITHUB_ENV + - run: echo "ACTOR=${{ github.event.review.user.login }}" >> $GITHUB_ENV - run: echo "REPOSITORY=${{ github.repository }}" >> $GITHUB_ENV - run: echo "REFS=${{ github.ref }}" >> $GITHUB_ENV - run: echo "PR_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV - run: echo "PUBLISH=true" >> $GITHUB_ENV - run: echo "$GITHUB_CONTEXT" + - run: echo "$GITHUB_ENV" - run: | - curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ env.TOKEN }}" https://api.github.com/repos/bytecodealliance/wasmtime-sightglass-benchmarking/dispatches -d '{"event_type":"bench-sightglass", "client_payload":{"message":"${{ env.MESSAGE }}", "actor":"${{ env.ACTOR }}", "repository":"${{ env.REPOSITORY }}", "ref":"${{ env.REFS }}", "pr_number":"${{ env.PR_NUMBER }}", "publish":"${{ env.PUBLISH }}" }' + # Create and Push Branch + git clone https://jlb6740:${{env.TOKEN}}@github.com/bytecodealliance/wasmtime-sightglass-benchmarking.git + cd wasmtime-sightglass-benchmarking + git remote add wasmtime ${{ github.event.repository.clone_url }} + git fetch wasmtime refs/pull/*/merge:refs/remotes/wasmtime/pull/*/merge + git checkout wasmtime/pull/${{ github.ref_name }} -b ${{ github.ref_name }} + git submodule update --init --recursive + git checkout -b wasmtime/${{ github.ref }}/${{ github.sha }} + sudo apt install jq + export commit_url=${{ github.event.pull_request._links.commits.href }} + echo $commit_url + echo $(curl -sSL $commit_url | jq -r '.[].commit.committer.name' | tail -n 1) + echo $(curl -sSL $commit_url | jq -r '.[].commit.committer.email' | tail -n 1) + git config user.name $(curl -sSL $commit_url | jq -r '.[].commit.committer.name' | tail -n 1) + git config user.email $(curl -sSL $commit_url | jq -r '.[].commit.committer.email' | tail -n 1) + git commit --allow-empty -m "${{ github.event.pull_request._links.comments.href }}" + git push origin wasmtime/${{ github.ref }}/${{ github.sha }} + #curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ env.TOKEN }}" https://api.github.com/repos/bytecodealliance/wasmtime-sightglass-benchmarking/dispatches -d '{"event_type":"Performance Testing", "client_payload":{"message":"${{ env.MESSAGE }}", "actor":"${{ env.ACTOR }}", "repository":"${{ env.REPOSITORY }}", "ref":"${{ env.REFS }}", "pr_number":"${{ env.PR_NUMBER }}", "publish":"${{ env.PUBLISH }}" }' - Sightglass_x86-64_from_workflow_dispatch: - name: Performance x86-64_from_workflow_dispatch - runs-on: ubuntu-latest - if: (github.event_name == 'workflow_dispatch') + Performance_Repo_On_Push: + name: Benchmark x64 on push Performance repo + runs-on: [self-hosted, linux, x64] + if: (github.event_name == 'push') && (github.repository == 'bytecodealliance/wasmtime-sightglass-benchmarking') steps: - - run: echo "MESSAGE=${{ github.event.inputs.message }}" >> $GITHUB_ENV - - run: echo "ACTOR=${{ github.event.inputs.actor }}" >> $GITHUB_ENV - - run: echo "REPOSITORY=${{ github.event.inputs.repository }}" >> $GITHUB_ENV - - run: echo "REFS=${{ github.event.inputs.refs }}" >> $GITHUB_ENV - - run: echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV - - run: echo "PUBLISH=${{ github.event.inputs.post_results }}" >> $GITHUB_ENV - run: echo "$GITHUB_CONTEXT" - - run: | - curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ env.TOKEN }}" https://api.github.com/repos/bytecodealliance/wasmtime-sightglass-benchmarking/dispatches -d '{"event_type":"bench-sightglass", "client_payload":{"message":"${{ env.MESSAGE }}", "actor":"${{ env.ACTOR }}", "repository":"${{ env.REPOSITORY }}", "ref":"${{ env.REFS }}", "pr_number":"${{ env.PR_NUMBER }}", "publish":"${{ env.PUBLISH }}" }' \ No newline at end of file + - run: echo "${{ github.event.head_commit.message }}" + - name: "Build sightglass commit '${{ env.SG_COMMIT }}'" + run: | + cd ../ && ls -l && rm -rf ./sightglass + git clone https://github.com/bytecodealliance/sightglass.git && cd ./sightglass + git checkout ${{env.SG_COMMIT}} + cargo build --release + + - name: Checkout patch from bytecodealliance/wasmtime + uses: actions/checkout@v3 + with: + submodules: true + repository: ${{ env.repository }} + ref: ${{ env.event.ref_name }} + path: wasmtime_commit + + - name: Build patch from bytecodealliance/wasmtime + working-directory: ./wasmtime_commit + run: | + cargo build --release -p wasmtime-bench-api + cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_commit.so + + - name: Checkout main from bytecodealliance/wasmtime + uses: actions/checkout@v3 + with: + ref: 'main' + repository: 'bytecodealliance/wasmtime' + submodules: true + path: wasmtime_main + + - name: Build main from bytecodealliance/wasmtime + working-directory: ./wasmtime_main + run: | + cargo build --release -p wasmtime-bench-api + cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_main.so + + - name: Run performance tests + working-directory: ../sightglass + run: | + cargo run -- \ + benchmark \ + --processes 1 \ + --iterations-per-process 2 \ + --engine /tmp/wasmtime_main.so \ + --engine /tmp/wasmtime_commit.so \ + --output-format csv \ + --output-file /tmp/results.csv \ + --raw \ + -- benchmarks/*/benchmark.wasm + ./target/release/sightglass-cli summarize --input-format csv --output-format csv -f /tmp/results.csv > /tmp/results_summarized.csv + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: Post Process Results + run: | + pip3 install pandas numpy + grep -v "nanoseconds" /tmp/results_summarized.csv > /tmp/results_cycles_summarized.csv + sed -i 's/\/tmp\/wasmtime_commit.so/patch/g' /tmp/results_cycles_summarized.csv + sed -i 's/\/tmp\/wasmtime_main.so/main/g' /tmp/results_cycles_summarized.csv + sed -i 's/benchmarks-next\///g' /tmp/results_cycles_summarized.csv + sed -i 's/\/benchmark.wasm//g' /tmp/results_cycles_summarized.csv + python3 -c "import pandas as pd; pp = pd.read_csv('/tmp/results_cycles_summarized.csv', \ + usecols=['arch','engine','wasm', 'phase', 'mean'], header=0); \ + pp_sorted = pp.sort_values(['wasm', 'phase', 'engine'], ascending=True); \ + pp_pct_changed=pp_sorted.groupby(['wasm','phase'])['mean'].pct_change().reset_index().rename(columns = {'mean':'pct_change'}); \ + pp_sorted.index.name = 'index'; \ + pp_sorted_merged=pp_sorted.merge(pp_pct_changed, on='index'); \ + pp_sorted_merged[pp_sorted_merged['engine'].str.contains('patch')]; \ + pp_sorted_merged=pp_sorted_merged[pp_sorted_merged['engine'].str.contains('patch')]; \ + pp_sorted_merged=pp_sorted_merged[['wasm','arch','phase','pct_change']]; \ + print(pp_sorted_merged.to_string(index=False));" > /tmp/results_cycles_summarized_sorted2.csv + sed -i 's/^/ /' /tmp/results_cycles_summarized_sorted2.csv + sed -i 's/ \+/|/g' /tmp/results_cycles_summarized_sorted2.csv + sed -i -z 's/\n/|\n/g' /tmp/results_cycles_summarized_sorted2.csv + sed -i '2 i\ |-|-|-|-|' /tmp/results_cycles_summarized_sorted2.csv + sed -i '/main/d' /tmp/results_cycles_summarized_sorted2.csv + sed -i '1 i\ Patch improvement over main/HEAD (x64). Negative is better.\n' /tmp/results_cycles_summarized_sorted2.csv + sed -i '1 i\ ${{ env.MESSAGE }}\n' /tmp/results_cycles_summarized_sorted2.csv + + - name: Print Results + run: cat /tmp/results_cycles_summarized_sorted2.csv + + - id: get-comment-body + name: Create Results Body + run: | + body="$(cat /tmp/results_cycles_summarized_sorted2.csv)" + body="${body//'%'/'%25'}" + body="${body//$'\n'/'%0A'}" + body="${body//$'\r'/'%0D'}" + echo "::set-output name=body::$body" + + - name: Publish Results + run: | + curl -X POST -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token ${{ secrets.WASMTIME_PUBLISHING_TOKEN }}" \ + ${{ github.event.head_commit.message }} \ + -d '{"body": ${{ toJSON(steps.get-comment-body.outputs.body) }}}' From 9326da192577e00220df6f0e0ec8965b3f5e0870 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Wed, 14 Sep 2022 10:08:19 -0700 Subject: [PATCH 03/13] Remove unused code --- .github/workflows/performance.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index c6b183e3f1dc..647b02b6dc79 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -4,7 +4,8 @@ name: "Performance Testing" # Controls when the action will run. -# This workflow runs when manually triggered using the UI or API. +# This workflow runs when manually triggered by keywords used in the start of a review comment +# Currently that phrase is /bench_x64. /bench_aarch64 and /bench_both are TODOs. on: pull_request_review: types: [submitted, edited] @@ -37,14 +38,7 @@ jobs: || ('sparker-arm' == github.event.review.user.login) || ('uweigand' == github.event.review.user.login)) steps: - - run: echo "MESSAGE=Requested from pull request comment." >> $GITHUB_ENV - - run: echo "ACTOR=${{ github.event.review.user.login }}" >> $GITHUB_ENV - - run: echo "REPOSITORY=${{ github.repository }}" >> $GITHUB_ENV - - run: echo "REFS=${{ github.ref }}" >> $GITHUB_ENV - - run: echo "PR_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV - - run: echo "PUBLISH=true" >> $GITHUB_ENV - run: echo "$GITHUB_CONTEXT" - - run: echo "$GITHUB_ENV" - run: | # Create and Push Branch git clone https://jlb6740:${{env.TOKEN}}@github.com/bytecodealliance/wasmtime-sightglass-benchmarking.git @@ -63,7 +57,6 @@ jobs: git config user.email $(curl -sSL $commit_url | jq -r '.[].commit.committer.email' | tail -n 1) git commit --allow-empty -m "${{ github.event.pull_request._links.comments.href }}" git push origin wasmtime/${{ github.ref }}/${{ github.sha }} - #curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ env.TOKEN }}" https://api.github.com/repos/bytecodealliance/wasmtime-sightglass-benchmarking/dispatches -d '{"event_type":"Performance Testing", "client_payload":{"message":"${{ env.MESSAGE }}", "actor":"${{ env.ACTOR }}", "repository":"${{ env.REPOSITORY }}", "ref":"${{ env.REFS }}", "pr_number":"${{ env.PR_NUMBER }}", "publish":"${{ env.PUBLISH }}" }' Performance_Repo_On_Push: name: Benchmark x64 on push Performance repo @@ -150,8 +143,7 @@ jobs: sed -i -z 's/\n/|\n/g' /tmp/results_cycles_summarized_sorted2.csv sed -i '2 i\ |-|-|-|-|' /tmp/results_cycles_summarized_sorted2.csv sed -i '/main/d' /tmp/results_cycles_summarized_sorted2.csv - sed -i '1 i\ Patch improvement over main/HEAD (x64). Negative is better.\n' /tmp/results_cycles_summarized_sorted2.csv - sed -i '1 i\ ${{ env.MESSAGE }}\n' /tmp/results_cycles_summarized_sorted2.csv + sed -i '1 i\ Patch improvement over main HEAD (x64). Negative shows reduction pct (improvement).\n' /tmp/results_cycles_summarized_sorted2.csv - name: Print Results run: cat /tmp/results_cycles_summarized_sorted2.csv From eaf93588116e42dc3e28b72ef01e6398131f4153 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 15 Sep 2022 10:50:13 -0700 Subject: [PATCH 04/13] Remvoes unused pull_request_review_comment trigger --- .github/workflows/performance.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 647b02b6dc79..f58354b0f54d 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -9,8 +9,6 @@ name: "Performance Testing" on: pull_request_review: types: [submitted, edited] - pull_request_review_comment: - types: [created, edited] push: # Env variables From 705f66cd53739f2f9d0e6eb91535719ecac0aace Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 15 Sep 2022 12:02:23 -0700 Subject: [PATCH 05/13] Match trigger word when contained anywhere in the pull request review message --- .github/workflows/performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index f58354b0f54d..4034d1f774ea 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest if: | (github.event_name == 'pull_request_review') && - (startsWith(github.event.review.body, '/bench_x64')) && + (contains(github.event.review.body, '/bench_x64')) && (('abrown' == github.event.review.user.login) || ('afonso360' == github.event.review.user.login) || ('akirilov-arm' == github.event.review.user.login) From 0678a0e56c0c3abb4916a7e561faa975d101fc19 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 15 Sep 2022 12:28:54 -0700 Subject: [PATCH 06/13] Remove redundant repo and ref variables for wasmtime_commit --- .github/workflows/performance.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 4034d1f774ea..3f68ac67f071 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -70,15 +70,13 @@ jobs: git checkout ${{env.SG_COMMIT}} cargo build --release - - name: Checkout patch from bytecodealliance/wasmtime + - name: Checkout patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo) uses: actions/checkout@v3 with: submodules: true - repository: ${{ env.repository }} - ref: ${{ env.event.ref_name }} path: wasmtime_commit - - name: Build patch from bytecodealliance/wasmtime + - name: Build patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo) working-directory: ./wasmtime_commit run: | cargo build --release -p wasmtime-bench-api From 551786ea13c9d0f5281ea201ac01d067928660d1 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 15 Sep 2022 12:48:32 -0700 Subject: [PATCH 07/13] Minor comment update --- .github/workflows/performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 3f68ac67f071..3a8a846444f5 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -5,7 +5,7 @@ name: "Performance Testing" # Controls when the action will run. # This workflow runs when manually triggered by keywords used in the start of a review comment -# Currently that phrase is /bench_x64. /bench_aarch64 and /bench_both are TODOs. +# Currently that phrase is /bench_x64. /bench_aarch64 and /bench_all are TODOs. on: pull_request_review: types: [submitted, edited] From 8f760fc9b61c5576cbcec5a3da1bd1b890919cae Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 15 Sep 2022 12:54:48 -0700 Subject: [PATCH 08/13] Remove command to install jq --- .github/workflows/performance.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 3a8a846444f5..6c1c5012b4e3 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -46,7 +46,6 @@ jobs: git checkout wasmtime/pull/${{ github.ref_name }} -b ${{ github.ref_name }} git submodule update --init --recursive git checkout -b wasmtime/${{ github.ref }}/${{ github.sha }} - sudo apt install jq export commit_url=${{ github.event.pull_request._links.commits.href }} echo $commit_url echo $(curl -sSL $commit_url | jq -r '.[].commit.committer.name' | tail -n 1) From 48f56fdd901ee0b8e88915b6a168504f022cd514 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 15 Sep 2022 12:59:39 -0700 Subject: [PATCH 09/13] Remove printing of git config variables being used --- .github/workflows/performance.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 6c1c5012b4e3..476bf6a70259 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -47,9 +47,6 @@ jobs: git submodule update --init --recursive git checkout -b wasmtime/${{ github.ref }}/${{ github.sha }} export commit_url=${{ github.event.pull_request._links.commits.href }} - echo $commit_url - echo $(curl -sSL $commit_url | jq -r '.[].commit.committer.name' | tail -n 1) - echo $(curl -sSL $commit_url | jq -r '.[].commit.committer.email' | tail -n 1) git config user.name $(curl -sSL $commit_url | jq -r '.[].commit.committer.name' | tail -n 1) git config user.email $(curl -sSL $commit_url | jq -r '.[].commit.committer.email' | tail -n 1) git commit --allow-empty -m "${{ github.event.pull_request._links.comments.href }}" From d1f2a380a02d54187b1180bcc50ce0ba6a731ec8 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 15 Sep 2022 13:17:23 -0700 Subject: [PATCH 10/13] Fix token for posting results --- .github/workflows/performance.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 476bf6a70259..3547beb2cc17 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -151,7 +151,9 @@ jobs: - name: Publish Results run: | + echo ${{ secrets.TOKEN }} + echo ${{ github.event.head_commit.message }} curl -X POST -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token ${{ secrets.WASMTIME_PUBLISHING_TOKEN }}" \ + -H "Authorization: token ${{ secrets.TOKEN }}" \ ${{ github.event.head_commit.message }} \ -d '{"body": ${{ toJSON(steps.get-comment-body.outputs.body) }}}' From 2ae992d8b226a4e8d4bafd30121cf8f8e6591ef3 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 15 Sep 2022 14:23:30 -0700 Subject: [PATCH 11/13] Update message explaining pct_change for benchmark results --- .github/workflows/performance.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 3547beb2cc17..8e70e54702f2 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -135,7 +135,10 @@ jobs: sed -i -z 's/\n/|\n/g' /tmp/results_cycles_summarized_sorted2.csv sed -i '2 i\ |-|-|-|-|' /tmp/results_cycles_summarized_sorted2.csv sed -i '/main/d' /tmp/results_cycles_summarized_sorted2.csv - sed -i '1 i\ Patch improvement over main HEAD (x64). Negative shows reduction pct (improvement).\n' /tmp/results_cycles_summarized_sorted2.csv + sed -i '1 i\ Shows pct_change on x64 for patch if merged compared to head for main.\n \ + Pct_change is based on clocktick event cycles where the benchmarks are run in Sightglass.\n \ + A negative pct_change for example means clockticks would be expected to be reduced for the benchmark, \ + for that phase, and by that factor, if the patch were merged (i.e. negative is good).\n' /tmp/results_cycles_summarized_sorted2.csv - name: Print Results run: cat /tmp/results_cycles_summarized_sorted2.csv From f6d7db91efae32a9c49ff280371386db55214d3f Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 15 Sep 2022 14:39:13 -0700 Subject: [PATCH 12/13] Revert TOKEN for publsh change --- .github/workflows/performance.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 8e70e54702f2..76429546775b 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -154,9 +154,7 @@ jobs: - name: Publish Results run: | - echo ${{ secrets.TOKEN }} - echo ${{ github.event.head_commit.message }} curl -X POST -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token ${{ secrets.TOKEN }}" \ + -H "Authorization: token ${{ secrets.WASMTIME_PUBLISHING_TOKEN }}" \ ${{ github.event.head_commit.message }} \ -d '{"body": ${{ toJSON(steps.get-comment-body.outputs.body) }}}' From 56673bcac43e331f83e19dbd6a974eda3753c40a Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Thu, 15 Sep 2022 14:56:38 -0700 Subject: [PATCH 13/13] Update message explaining results --- .github/workflows/performance.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 76429546775b..f7b046cc43e5 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -135,9 +135,9 @@ jobs: sed -i -z 's/\n/|\n/g' /tmp/results_cycles_summarized_sorted2.csv sed -i '2 i\ |-|-|-|-|' /tmp/results_cycles_summarized_sorted2.csv sed -i '/main/d' /tmp/results_cycles_summarized_sorted2.csv - sed -i '1 i\ Shows pct_change on x64 for patch if merged compared to head for main.\n \ - Pct_change is based on clocktick event cycles where the benchmarks are run in Sightglass.\n \ - A negative pct_change for example means clockticks would be expected to be reduced for the benchmark, \ + sed -i '1 i\Shows pct_change on x64 for the patch if merged compared to current head for main.\n\ + Pct_change is based on clocktick event cycles where the benchmarks are run with Sightglass. \ + A negative pct_change means clockticks are expected to be reduced for the benchmark, \ for that phase, and by that factor, if the patch were merged (i.e. negative is good).\n' /tmp/results_cycles_summarized_sorted2.csv - name: Print Results