From 8b5faa4251711d2ae6e439fda058329c343a8215 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sat, 27 Jul 2024 14:10:53 +0400 Subject: [PATCH 1/4] Separate benchmark run and results uploading --- .github/workflows/benchmark-upload.yml | 77 ++++++++++++++++++++++++++ .github/workflows/benchmark.yml | 68 ----------------------- 2 files changed, 77 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/benchmark-upload.yml diff --git a/.github/workflows/benchmark-upload.yml b/.github/workflows/benchmark-upload.yml new file mode 100644 index 00000000..1e529c28 --- /dev/null +++ b/.github/workflows/benchmark-upload.yml @@ -0,0 +1,77 @@ +name: Upload and compare benchmark results + +on: + workflow_run: + types: [completed] + workflows: ['Running benchmark'] + +jobs: + upload-benchmark-results: + if: (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'pull_request') && github.repository == 'OptimumCode/json-schema-validator' + runs-on: ubuntu-latest + environment: contribution + env: + RESULTS_DIR: bench-results + permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write + # pull-requests permission to create comments on PR in case of alert + pull-requests: write + strategy: + # to make sure results are submitted one by one + max-parallel: 1 + matrix: + include: + - artifact-pattern: 'bench-result-*' + results-name: KMP JSON schema validator + alert: true + - artifact-pattern: 'bench-comparison-result-*' + results-name: Compare KMP JSON schema validator + alert: false + name: 'Process benchmark results for ${{ matrix.results-name }}' + steps: + - name: 'Checkout Repository' + uses: actions/checkout@v4 + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + pattern: ${{ matrix.artifact-pattern }} + path: ${{ env.RESULTS_DIR }} + merge-multiple: true + run-id: ${{ github.event.workflow_run.id }} + - name: Show downloaded artifacts + run: tree ${{ env.RESULTS_DIR }} + - name: Prepare and join benchmark reports + id: prep + run: | + for report in $(find ./${{ env.RESULTS_DIR }} -type f -name "*.json") + do + file_name=$(basename "$report") + platform="${file_name%.*}" + jq "[ .[] | .benchmark |= \"${platform}.\" + ltrimstr(\"io.github.optimumcode.json.schema.benchmark.\") | .params |= map_values(. |= split(\"/\")[-1]) ]" $report > ${{ env.RESULTS_DIR }}/$platform.json + done + AGGREGATED_REPORT=aggregated.json + # Joined reports looks like this: [[{},{}], [{},{}]] + # We need to transform them into this: [{},{}] + ls ${{ env.RESULTS_DIR }}/*.json + jq -s '[ .[] | .[] ]' ${{ env.RESULTS_DIR }}/*.json > $AGGREGATED_REPORT + echo "report=$AGGREGATED_REPORT" >> $GITHUB_OUTPUT + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: ${{ matrix.results-name }} + tool: 'jmh' + output-file-path: ${{ steps.prep.outputs.report }} + alert-comment-cc-users: "@OptimumCode" + comment-on-alert: ${{ matrix.alert }} + summary-always: true + alert-threshold: '150%' + fail-threshold: '200%' + max-items-in-chart: 50 + github-token: ${{ secrets.GITHUB_TOKEN }} + # Push and deploy GitHub pages branch automatically only if run in main repo and not in PR + auto-push: ${{ github.event.workflow_run.event == 'push' }} + + diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ba4e521f..7102bc00 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -138,71 +138,3 @@ jobs: with: name: bench-comparison-result-${{ matrix.os }} path: ${{ env.BENCHMARK_RESULTS }}/comparison/**/*.json - - upload-benchmark-results: - if: (github.event_name == 'push' || github.event_name == 'pull_request') && github.repository == 'OptimumCode/json-schema-validator' - needs: - - benchmark-matrix - runs-on: ubuntu-latest - env: - RESULTS_DIR: bench-results - permissions: - # deployments permission to deploy GitHub pages website - deployments: write - # contents permission to update benchmark contents in gh-pages branch - contents: write - # pull-requests permission to create comments on PR in case of alert - pull-requests: write - strategy: - # to make sure results are submitted one by one - max-parallel: 1 - matrix: - include: - - artifact-pattern: 'bench-result-*' - results-name: KMP JSON schema validator - alert: true - - artifact-pattern: 'bench-comparison-result-*' - results-name: Compare KMP JSON schema validator - alert: false - name: 'Process benchmark results for ${{ matrix.results-name }}' - steps: - - name: 'Checkout Repository' - uses: actions/checkout@v4 - - name: Download benchmark results - uses: actions/download-artifact@v4 - with: - pattern: ${{ matrix.artifact-pattern }} - path: ${{ env.RESULTS_DIR }} - merge-multiple: true - - name: Show downloaded artifacts - run: tree ${{ env.RESULTS_DIR }} - - name: Prepare and join benchmark reports - id: prep - run: | - for report in $(find ./${{ env.RESULTS_DIR }} -type f -name "*.json") - do - file_name=$(basename "$report") - platform="${file_name%.*}" - jq "[ .[] | .benchmark |= \"${platform}.\" + ltrimstr(\"io.github.optimumcode.json.schema.benchmark.\") | .params |= map_values(. |= split(\"/\")[-1]) ]" $report > ${{ env.RESULTS_DIR }}/$platform.json - done - AGGREGATED_REPORT=aggregated.json - # Joined reports looks like this: [[{},{}], [{},{}]] - # We need to transform them into this: [{},{}] - ls ${{ env.RESULTS_DIR }}/*.json - jq -s '[ .[] | .[] ]' ${{ env.RESULTS_DIR }}/*.json > $AGGREGATED_REPORT - echo "report=$AGGREGATED_REPORT" >> $GITHUB_OUTPUT - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: ${{ matrix.results-name }} - tool: 'jmh' - output-file-path: ${{ steps.prep.outputs.report }} - alert-comment-cc-users: "@OptimumCode" - comment-on-alert: ${{ matrix.alert }} - summary-always: true - alert-threshold: '150%' - fail-threshold: '200%' - max-items-in-chart: 50 - github-token: ${{ secrets.GITHUB_TOKEN }} - # Push and deploy GitHub pages branch automatically only if run in main repo and not in PR - auto-push: ${{ github.event_name != 'pull_request' }} From 1ac2239c6db37ab0a410e477f0b5df3fef28e0f9 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sat, 27 Jul 2024 14:47:24 +0400 Subject: [PATCH 2/4] Try to remove environment --- .github/workflows/benchmark-upload.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/benchmark-upload.yml b/.github/workflows/benchmark-upload.yml index 1e529c28..05ee2464 100644 --- a/.github/workflows/benchmark-upload.yml +++ b/.github/workflows/benchmark-upload.yml @@ -9,7 +9,6 @@ jobs: upload-benchmark-results: if: (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'pull_request') && github.repository == 'OptimumCode/json-schema-validator' runs-on: ubuntu-latest - environment: contribution env: RESULTS_DIR: bench-results permissions: From 3d32129e4fbefe2deb44e17f44031db7421e8bb1 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sat, 27 Jul 2024 15:39:30 +0400 Subject: [PATCH 3/4] Extract code covarage upload --- .github/workflows/build-and-test.yml | 6 ++++ .github/workflows/check.yml | 11 +++++++ .github/workflows/code-coverage-upload.yml | 37 ++++++++++++++++++++++ .github/workflows/pull_request.yml | 17 ++++++++-- 4 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/code-coverage-upload.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 0d3c65aa..8de4ec14 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -8,6 +8,11 @@ on: description: "enabled uploading coverage report to codecov" required: false default: false + collect-code-coverage: + type: boolean + description: "enables collecting coverage reports and uploading them as artifacts" + required: false + default: false secrets: CODECOV_TOKEN: description: "token to upload codecov report" @@ -50,6 +55,7 @@ jobs: run-on: ubuntu-latest task: linuxAllTest upload-code-coverage: ${{ inputs.upload-code-coverage }} + collect-code-coverage: ${{ inputs.collect-code-coverage }} secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} check-macos: diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e92243a9..45a0f59a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -16,6 +16,11 @@ on: description: "enabled uploading coverage report to codecov" required: false default: false + collect-code-coverage: + type: boolean + description: "enables collecting coverage reports and uploading them as artifacts" + required: false + default: false secrets: CODECOV_TOKEN: description: "token to upload codecov report" @@ -66,3 +71,9 @@ jobs: uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} + - name: Collect coverage reports + if: inputs.collect-code-coverage && github.actor != 'dependabot[bot]' + uses: actions/upload-artifact@v4 + with: + name: coverage-reports + path: '**/build/reports/kover/report.xml' diff --git a/.github/workflows/code-coverage-upload.yml b/.github/workflows/code-coverage-upload.yml new file mode 100644 index 00000000..d22d2968 --- /dev/null +++ b/.github/workflows/code-coverage-upload.yml @@ -0,0 +1,37 @@ +name: Upload code coverage results + +on: + workflow_run: + workflows: ['Check the PR'] + types: [completed] + +jobs: + upload: + runs-on: ubuntu-latest + if: github.event.workflow_run.actor.name != 'dependabot[bot]' + steps: + - name: 'Checkout Repository' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_branch }} + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: coverage-reports + path: reports/ + run-id: ${{ github.event.workflow_run.id }} + - name: Download PR number + uses: actions/download-artifact@v4 + with: + name: pr-number + path: '.' + run-id: ${{ github.event.workflow_run.id }} + - id: trigger + run: echo "pr-number=$(cat pr_number)" >> GITHUB_OUTPUT + - name: Test pull_requests objects + run: echo ${{ github.event.workflow_run.pull_requests }} + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4 + with: + override_pr: ${{ steps.trigger.outputs.pr-number }} + token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 175e316d..69ecb3a9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -53,9 +53,7 @@ jobs: check-pr: uses: ./.github/workflows/build-and-test.yml with: - upload-code-coverage: true - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + collect-code-coverage: true danger-check: runs-on: ubuntu-latest permissions: @@ -71,4 +69,17 @@ jobs: dangerfile: Dangerfile.df.kts env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + store-pr-number: + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + mkdir -p ./pr + echo $PR_NUMBER > ./pr/pr_number + - uses: actions/upload-artifact@v4 + with: + name: pr-number + path: pr/ From e1d2eb3260d958a823ee783d96288532059e86ff Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sat, 27 Jul 2024 15:41:02 +0400 Subject: [PATCH 4/4] Add repository check --- .github/workflows/code-coverage-upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-coverage-upload.yml b/.github/workflows/code-coverage-upload.yml index d22d2968..831f0ace 100644 --- a/.github/workflows/code-coverage-upload.yml +++ b/.github/workflows/code-coverage-upload.yml @@ -8,7 +8,7 @@ on: jobs: upload: runs-on: ubuntu-latest - if: github.event.workflow_run.actor.name != 'dependabot[bot]' + if: github.event.workflow_run.actor.name != 'dependabot[bot]' && github.repository == 'OptimumCode/json-schema-validator' steps: - name: 'Checkout Repository' uses: actions/checkout@v4