[storage] split the replay_verify to multiple jobs #6
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "*run replay-verify reusable workflow" | ||
| on: | ||
| # This allows the workflow to be triggered from another workflow | ||
| workflow_call: | ||
| inputs: | ||
| GIT_SHA: | ||
| required: true | ||
| type: string | ||
| description: The git SHA1 to test. | ||
| # replay-verify config | ||
| BUCKET: | ||
| required: true | ||
| type: string | ||
| description: The bucket to use for the backup. If not specified, it will use the default bucket. | ||
| SUB_DIR: | ||
| required: true | ||
| type: string | ||
| description: The subdirectory to use for the backup. If not specified, it will use the default subdirectory. | ||
| HISTORY_START: | ||
| required: true | ||
| type: string | ||
| description: The history start to use for the backup. If not specified, it will use the default history start. | ||
| TXNS_TO_SKIP: | ||
| required: false | ||
| type: string | ||
| description: The list of transaction versions to skip. If not specified, it will use the default list. | ||
| BACKUP_CONFIG_TEMPLATE_PATH: | ||
| description: "The path to the backup config template to use." | ||
| type: string | ||
| required: true | ||
| # GHA job config | ||
| RUNS_ON: | ||
| description: "The runner to use for the job." | ||
| type: string | ||
| required: true | ||
| default: "high-perf-docker-with-local-ssd" | ||
| TIMEOUT_MINUTES: | ||
| description: "Github job timeout in minutes" | ||
| type: number | ||
| required: true | ||
| default: 720 | ||
| # This allows the workflow to be triggered manually from the Github UI or CLI | ||
| # NOTE: because the "number" type is not supported, we default to 720 minute timeout | ||
| workflow_dispatch: | ||
| inputs: | ||
| GIT_SHA: | ||
| required: true | ||
| type: string | ||
| description: The git SHA1 to test. | ||
| # replay-verify config | ||
| BUCKET: | ||
| required: true | ||
| type: string | ||
| description: The bucket to use for the backup. If not specified, it will use the default bucket. | ||
| SUB_DIR: | ||
| required: true | ||
| type: string | ||
| description: The subdirectory to use for the backup. If not specified, it will use the default subdirectory. | ||
| HISTORY_START: | ||
| required: true | ||
| type: string | ||
| description: The history start to use for the backup. If not specified, it will use the default history start. | ||
| TXNS_TO_SKIP: | ||
| required: false | ||
| type: string | ||
| description: The list of transaction versions to skip. If not specified, it will use the default list. | ||
| BACKUP_CONFIG_TEMPLATE_PATH: | ||
| description: "The path to the backup config template to use." | ||
| type: string | ||
| required: true | ||
| # GHA job config | ||
| RUNS_ON: | ||
| description: "The runner to use for the job." | ||
| type: string | ||
| required: true | ||
| default: "high-perf-docker-with-local-ssd" | ||
| jobs: | ||
| generate-runner: | ||
| runs-on: ${{ inputs.RUNS_ON }} | ||
| outputs: | ||
| matrix: ${{ steps.set-runner-number.outputs.matrix }} | ||
| steps: | ||
| id: set-runner-number | ||
| run: | | ||
| echo "::set-output name=matrix::{\"include\": [$(for i in {1..10}; do echo \{\"number\":$i\}; if [ $i -lt 10 ]; then echo ','; fi; done)]}" # 10 runners in parallel | ||
| replay-verify: | ||
| needs: generate-runner | ||
| timeout-minutes: ${{ inputs.TIMEOUT_MINUTES || 720 }} | ||
| runs-on: ${{ inputs.RUNS_ON }} | ||
| strategy: | ||
| matrix: ${{fromJson(needs.generate-runner.outputs.matrix)}} # 10 runners in parallel | ||
| steps: | ||
| - name: Echo Runner Number | ||
| run: echo "Number is ${{ matrix.number }}" | ||
| - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3 | ||
| with: | ||
| ref: ${{ inputs.GIT_SHA }} | ||
| - uses: aptos-labs/aptos-core/.github/actions/rust-setup@main | ||
| with: | ||
| GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }} | ||
| - name: Install AWS CLI | ||
| shell: bash | ||
| run: | | ||
| scripts/dev_setup.sh -b -i awscli | ||
| echo "${HOME}/bin/" >> $GITHUB_PATH # default INSTALL_DIR to path | ||
| - name: Build CLI binaries in release mode | ||
| shell: bash | ||
| run: cargo build --release -p aptos-db-tool | ||
| - name: Run replay-verify in parallel | ||
| shell: bash | ||
| run: testsuite/replay_verify.py ${{ matrix.number }} 10 # first argument is the runner number, second argument is the total number of runners | ||
| env: | ||
| BUCKET: ${{ inputs.BUCKET }} | ||
| SUB_DIR: ${{ inputs.SUB_DIR }} | ||
| HISTORY_START: ${{ inputs.HISTORY_START }} | ||
| TXNS_TO_SKIP: ${{ inputs.TXNS_TO_SKIP }} | ||
| BACKUP_CONFIG_TEMPLATE_PATH: ${{ inputs.BACKUP_CONFIG_TEMPLATE_PATH }} | ||
