QA - RPC Performance Tests #274
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: QA - RPC Performance Tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 0 * * 1-6' # Runs every day from Monday to Saturday at 00:00 AM UTC | |
| jobs: | |
| performance-test-suite: | |
| strategy: | |
| matrix: | |
| backend: [ Erigon2 ] | |
| runs-on: [ self-hosted, "${{ matrix.backend }}" ] | |
| env: | |
| ERIGON_DIR: /opt/erigon-versions/reference-version | |
| ERIGON_DATA_DIR: /opt/erigon-versions/reference-version/datadir | |
| RPC_PAST_TEST_DIR: /opt/rpc-past-tests | |
| ERIGON_QA_PATH: /opt/erigon-qa | |
| steps: | |
| - name: Checkout Silkworm repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| fetch-depth: "0" | |
| - name: Checkout RPC Tests Repository & Install Requirements | |
| run: | | |
| rm -rf ${{runner.workspace}}/rpc-tests | |
| git -c advice.detachedHead=false clone --depth 1 --branch v0.26.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests | |
| cd ${{runner.workspace}}/rpc-tests | |
| pip3 install -r requirements.txt | |
| - name: Clean Build Directory | |
| run: rm -rf ${{runner.workspace}}/silkworm/build | |
| - name: Create Build Environment | |
| run: cmake -E make_directory ${{runner.workspace}}/silkworm/build | |
| - name: Configure CMake | |
| working-directory: ${{runner.workspace}}/silkworm/build | |
| run: | | |
| cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=Release | |
| - name: Build Silkworm RpcDaemon | |
| working-directory: ${{runner.workspace}}/silkworm/build | |
| run: cmake --build . --config Release --target rpcdaemon -j 8 | |
| - name: Pause the Erigon instance dedicated to db maintenance | |
| run: | | |
| python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true | |
| - name: Run Silkworm RpcDaemon | |
| working-directory: ${{runner.workspace}}/silkworm/build/cmd | |
| run: | | |
| ./rpcdaemon --datadir $ERIGON_DATA_DIR --api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --log.verbosity 1 --erigon_compatibility --jwt ./jwt.hex --skip_protocol_check --http.compression --eth.addr 127.0.0.1:51515 & | |
| SILKWORM_RPC_DAEMON_PID=$! | |
| echo "SILKWORM_RPC_DAEMON_PID=$SILKWORM_RPC_DAEMON_PID" >> $GITHUB_ENV | |
| - name: Run Erigon RpcDaemon | |
| run: | | |
| $ERIGON_DIR/rpcdaemon --datadir $ERIGON_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --verbosity 1 & | |
| ERIGON_RPC_DAEMON_PID=$! | |
| echo "ERIGON_RPC_DAEMON_PID=$ERIGON_RPC_DAEMON_PID" >> $GITHUB_ENV | |
| - name: Run RPC Performance Tests | |
| id: test_step | |
| run: | | |
| set +e # Disable exit on error | |
| failed_test=0 | |
| commit=$(git -C ${{runner.workspace}}/silkworm rev-parse --short HEAD) # use ${{ github.sha }} or GITHUB_SHA | |
| past_test_dir=$RPC_PAST_TEST_DIR/mainnet_$(date +%Y%m%d_%H%M%S)_perf_$commit | |
| echo "past_test_dir=$past_test_dir" >> $GITHUB_ENV | |
| # Prepare historical test results directory | |
| mkdir -p $past_test_dir | |
| rm -rf $RPC_PAST_TEST_DIR/mainnet_bin # we want only the latest binary files | |
| mkdir -p $RPC_PAST_TEST_DIR/mainnet_bin | |
| run_perf () { | |
| servers=("silkworm" "erigon") | |
| for i in 1 2 | |
| do | |
| network=$1 | |
| method=$2 | |
| pattern=$3 | |
| sequence=$4 | |
| # clean temporary area | |
| cd ${{runner.workspace}}/rpc-tests/perf | |
| rm -rf ./reports/ | |
| python3 ./run_perf_tests.py --blockchain "$network" \ | |
| --test-type "$method" \ | |
| --pattern-file pattern/"$network"/"$pattern".tar \ | |
| --test-sequence "$sequence" \ | |
| --repetitions 5 \ | |
| --silk-dir ${{runner.workspace}}/silkworm \ | |
| --erigon-dir $ERIGON_DATA_DIR \ | |
| --test-mode $i \ | |
| --test-report \ | |
| --json-report ./reports/mainnet/result.json \ | |
| --testing-daemon ${servers[i-1]} | |
| # Capture test runner script exit status | |
| perf_exit_status=$? | |
| # Preserve test results | |
| mv ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/result.json ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/${servers[i-1]}-$method-result.json | |
| # Detect the pre-built db version | |
| db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_DIR/production.ini production erigon_repo_commit) | |
| # Check test runner script exit status | |
| if [ $perf_exit_status -eq 0 ]; then | |
| # save all vegeta binary report | |
| echo "Save current vegeta binary files" | |
| cp -r ${{runner.workspace}}/rpc-tests/perf/reports/bin $RPC_PAST_TEST_DIR/mainnet_bin | |
| echo "Save test result on DB" | |
| cd ${{runner.workspace}}/silkworm | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \ | |
| --repo silkworm \ | |
| --branch ${{ github.ref_name }} \ | |
| --commit $(git rev-parse HEAD) \ | |
| --test_name rpc-performance-test-${servers[i-1]}-$method \ | |
| --chain mainnet \ | |
| --runner ${{ runner.name }} \ | |
| --db_version $db_version \ | |
| --outcome success \ | |
| --result_file ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/${servers[i-1]}-$method-result.json | |
| if [ $? -ne 0 ]; then | |
| failed_test=1 | |
| echo "Failure saving test results on DB" | |
| fi | |
| echo "Execute Latency Percentile HDR Analysis" | |
| cd ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/ | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/rpc-tests/perf_hdr_analysis.py \ | |
| --test_name ${servers[i-1]}-$method \ | |
| --input_file ./result.json \ | |
| --output_file ./${servers[i-1]}-${method}-latency_hdr_analysis.pdf | |
| else | |
| failed_test=1 | |
| cd ${{runner.workspace}}/silkworm | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \ | |
| --repo silkworm \ | |
| --branch ${{ github.ref_name }} \ | |
| --commit $(git rev-parse HEAD) \ | |
| --test_name rpc-performance-test-${servers[i-1]}-$method \ | |
| --chain mainnet \ | |
| --runner ${{ runner.name }} \ | |
| --db_version $db_version \ | |
| --outcome failure | |
| fi | |
| # Save test results to a directory with timestamp and commit hash | |
| cp -r ${{runner.workspace}}/rpc-tests/perf/reports/mainnet $past_test_dir | |
| done | |
| } | |
| # Launch the RPC performance test runner | |
| failed_test= 0 | |
| run_perf mainnet eth_call stress_test_eth_call_001_14M 1:1,100:30,1000:20,10000:20,20000:20 | |
| run_perf mainnet eth_getLogs stress_test_eth_getLogs_15M 1:1,100:30,1000:20,10000:20,20000:20 | |
| run_perf mainnet eth_getBalance stress_test_eth_getBalance_15M 1:1,100:30,1000:20,10000:20,20000:20 | |
| run_perf mainnet eth_getBlockByHash stress_test_eth_getBlockByHash_14M 1:1,100:30,1000:20,10000:20 | |
| run_perf mainnet eth_getBlockByNumber stress_test_eth_getBlockByNumber_13M 1:1,100:30,1000:20,5000:20 | |
| run_perf mainnet eth_getTransactionByHash stress_test_eth_getTransactionByHash_13M 1:1,100:30,1000:20,10000:20 | |
| run_perf mainnet eth_getTransactionReceipt stress_test_eth_getTransactionReceipt_14M 1:1,100:30,1000:20,5000:20,10000:20,20000:20 | |
| run_perf mainnet eth_createAccessList stress_test_eth_createAccessList_16M 1:1,100:30,1000:20,10000:20,20000:20 | |
| if [ $failed_test -eq 0 ]; then | |
| echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT" | |
| echo "Tests completed successfully" | |
| else | |
| echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" | |
| echo "Error detected during tests" | |
| fi | |
| - name: Stop Silkworm RpcDaemon | |
| working-directory: ${{runner.workspace}}/silkworm/build/cmd | |
| run: | | |
| # Clean up process if it's still running | |
| if kill -0 $SILKWORM_RPC_DAEMON_PID 2> /dev/null; then | |
| echo "Terminating Silkworm RpcDaemon" | |
| kill $SILKWORM_RPC_DAEMON_PID | |
| else | |
| echo "Silkworm RpcDaemon has already terminated" | |
| fi | |
| - name: Stop Erigon RpcDaemon | |
| run: | | |
| # Clean up process if it's still running | |
| if kill -0 $ERIGON_RPC_DAEMON_PID 2> /dev/null; then | |
| echo "Terminating Erigon RpcDaemon" | |
| kill $ERIGON_RPC_DAEMON_PID | |
| else | |
| echo "Erigon RpcDaemon has already terminated" | |
| fi | |
| - name: Resume the Erigon instance dedicated to db maintenance | |
| run: | | |
| python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true | |
| - name: Run change point analysis | |
| if: steps.test_step.outputs.TEST_RESULT == 'success' | |
| working-directory: ${{runner.workspace}}/rpc-tests/perf/reports/mainnet | |
| run: | | |
| set +e # Disable exit on error | |
| open_change_points=0 | |
| python3 $ERIGON_QA_PATH/test_system/qa-tests/change-points/change_point_analysis.py | |
| open_change_points=$? | |
| cp change_point_analysis.pdf $past_test_dir | |
| if [ $open_change_points -ne 0 ]; then | |
| echo "Change point analysis found points that need to be investigated" | |
| #echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" -- enable in the future | |
| fi | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results | |
| path: ${{ env.past_test_dir }} | |
| - name: Action for Success | |
| if: steps.test_step.outputs.TEST_RESULT == 'success' | |
| run: echo "::notice::Tests completed successfully" | |
| - name: Action for Not Success | |
| if: steps.test_step.outputs.TEST_RESULT != 'success' | |
| run: | | |
| echo "::error::Error detected during tests" | |
| exit 1 |