diff --git a/.github/workflows/load-tests.yml b/.github/workflows/load-tests.yml new file mode 100644 index 000000000..4bda65fd4 --- /dev/null +++ b/.github/workflows/load-tests.yml @@ -0,0 +1,186 @@ +name: Load Tests + +on: + push: + branches: [main, development] + pull_request: + branches: [main, development] + +jobs: + test-base: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.base.sha }} + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: '20' + + - name: Install dependencies + run: npm ci + + - name: Initialize the project + run: npm run init + + - name: Build + run: npm run build + + - name: Copy devnet config file from src to dist + run: cp ./config/config.devnet.yaml ./dist/config/config.yaml + + - name: Start docker services + run: docker compose up -d + + - name: Start Node.js API + run: node ./dist/src/main.js & + + - name: Install k6 + run: | + sudo gpg -k + sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 + echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list + sudo apt-get update + sudo apt-get install k6 + + - name: Wait for API to be ready + run: | + until curl --output /dev/null --silent --fail http://localhost:4001/hello; do + echo 'Waiting for API...' + sleep 1 + done + + - name: Preload cache + run: k6 run ./k6/preload.js + + - name: Run k6 Load Test + run: k6 run ./k6/script.js + + - name: Upload result file for base branch + uses: actions/upload-artifact@v3 + with: + name: base-results + path: k6/output/summary.json + + - name: Stop docker services + run: docker compose down + + test-head: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: '20' + + - name: Install dependencies + run: npm ci + + - name: Initialize the project + run: npm run init + + - name: Build + run: npm run build + + - name: Copy devnet config file from src to dist + run: cp ./config/config.devnet.yaml ./dist/config/config.yaml + + - name: Start docker services + run: docker compose up -d + + - name: Start Node.js API + run: node ./dist/src/main.js & + + - name: Install k6 + run: | + sudo gpg -k + sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 + echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list + sudo apt-get update + sudo apt-get install k6 + + - name: Wait for API to be ready + run: | + until curl --output /dev/null --silent --fail http://localhost:4001/hello; do + echo 'Waiting for API...' + sleep 1 + done + + - name: Preload cache + run: k6 run ./k6/preload.js + + - name: Run k6 Load Test + run: k6 run ./k6/script.js + + - name: Upload result file for head branch + uses: actions/upload-artifact@v3 + with: + name: head-results + path: k6/output/summary.json + + - name: Stop docker services + run: docker compose down + + compare-results: + runs-on: ubuntu-latest + + needs: [test-base, test-head] + steps: + - uses: actions/checkout@v2 + + - name: Download all artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: '20' + + - name: Compare test results + run: | + node ./k6/compare-results.js ${{ github.event.pull_request.base.sha }} artifacts/base-results/summary.json ${{ github.event.pull_request.head.sha }} artifacts/head-results/summary.json report.md + + - name: Render the report from the template + id: template + uses: chuhlomin/render-template@v1 + if: github.event_name == 'pull_request' + with: + template: report.md + vars: | + base: ${{ github.event.pull_request.base.sha }} + head: ${{ github.event.pull_request.head.sha }} + + - name: Upload the report markdown + uses: actions/upload-artifact@v3 + if: github.event_name == 'pull_request' + with: + name: report-markdown + path: report.md + + - name: Find the comment containing the report + id: fc + uses: peter-evans/find-comment@v2 + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: 'k6 load testing comparison' + + - name: Create or update the report comment + uses: peter-evans/create-or-update-comment@v2 + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: ${{ steps.template.outputs.result }} + edit-mode: replace diff --git a/k6/.gitignore b/k6/.gitignore new file mode 100644 index 000000000..77d1aa0a2 --- /dev/null +++ b/k6/.gitignore @@ -0,0 +1,2 @@ +output/* +!output/.gitkeep \ No newline at end of file diff --git a/k6/compare-results.js b/k6/compare-results.js new file mode 100644 index 000000000..c7be7661c --- /dev/null +++ b/k6/compare-results.js @@ -0,0 +1,124 @@ +const fs = require('fs'); + +function generateComparisonTable(baseCommitHash, baseMetricsPath, targetCommitHash, targetMetricsPath, outputPath) { + // Load JSON outputs from k6 + const baseMetrics = JSON.parse(fs.readFileSync(baseMetricsPath, 'utf8')); + const targetMetrics = JSON.parse(fs.readFileSync(targetMetricsPath, 'utf8')); + + const baseData = extractMetrics(baseMetrics); + const targetData = extractMetrics(targetMetrics); + + const table = generateTable(baseCommitHash, baseData, targetCommitHash, targetData); + + fs.writeFileSync(outputPath, table); +} + +function extractMetrics(metrics) { + const extractedMetrics = {}; + const metricKeys = Object.keys(metrics.metrics); + + for (const key of metricKeys) { + if (key.endsWith('_http_req_duration')) { + const values = metrics.metrics[key].values; + const avgResponseTime = values.avg; + const maxResponseTime = values.max; + const p90 = values['p(90)']; + const p95 = values['p(95)']; + + const name = key.split('_')[0].charAt(0).toUpperCase() + key.split('_')[0].slice(1); + + if (!extractedMetrics[name]) { + extractedMetrics[name] = { avgResponseTime, maxResponseTime, p90, p95 }; + } else { + extractedMetrics[name].avgResponseTime = avgResponseTime; + extractedMetrics[name].maxResponseTime = maxResponseTime; + extractedMetrics[name].p90 = p90; + extractedMetrics[name].p95 = p95; + } + } + } + + extractedMetrics['Test Run Duration'] = metrics.state.testRunDurationMs; + + return extractedMetrics; +} + +function generateTable(baseCommitHash, baseData, targetCommitHash, targetData) { + const headers = ['Avg', 'Max', '90', '95']; + let table = `k6 load testing comparison.\nBase Commit Hash: ${baseCommitHash}\nTarget Commit Hash: ${targetCommitHash}\n\n`; + table += ' \ + \ + \ + \ + \ + ' + for (let i = 0; i < 3; i++) { + headers.forEach(header => { + table += ``; + }); + } + table += ''; + + for (const key of Object.keys(baseData)) { + if (key === 'Test Run Duration') { + continue; + } + const baseAvg = baseData[key].avgResponseTime; + const targetAvg = targetData[key].avgResponseTime; + const baseMax = baseData[key].maxResponseTime; + const targetMax = targetData[key].maxResponseTime; + const baseP90 = baseData[key].p90; + const targetP90 = targetData[key].p90; + const baseP95 = baseData[key].p95; + const targetP95 = targetData[key].p95; + + const avgDiff = getDifferencePercentage(baseAvg, targetAvg); + const maxDiff = getDifferencePercentage(baseMax, targetMax); + const p90Diff = getDifferencePercentage(baseP90, targetP90); + const p95Diff = getDifferencePercentage(baseP95, targetP95); + + const avgColor = getColor(baseAvg, targetAvg); + const maxColor = getColor(baseMax, targetMax); + const p90Color = getColor(baseP90, targetP90); + const p95Color = getColor(baseP95, targetP95); + + table += ``; + table += ``; + table += ``; + table += ``; + } + + const baseDuration = baseData['Test Run Duration'].toFixed(2); + const targetDuration = targetData['Test Run Duration'].toFixed(2); + table += `
MetricBaseTargetDiff
${header}
${key}${baseAvg.toFixed(2)}${baseMax.toFixed(2)}${baseP90.toFixed(2)}${baseP95.toFixed(2)}${targetAvg.toFixed(2)}${targetMax.toFixed(2)}${targetP90.toFixed(2)}${targetP95.toFixed(2)}${avgDiff} ${avgColor}${maxDiff} ${maxColor}${p90Diff} ${p90Color}${p95Diff} ${p95Color}
Test Run Duration${baseDuration}${targetDuration}
`; + table += '\n\nLegend: Avg - Average Response Time, Max - Maximum Response Time, 90 - 90th Percentile, 95 - 95th Percentile\nAll times are in milliseconds.\n'; + + return table; +} + +function getColor(baseValue, targetValue) { + if (baseValue >= targetValue) { + return '✅'; // Green emoji for improvement or equivalence + } else { + return '🔴'; // Red emoji for degradation + } +} + +function getDifferencePercentage(baseValue, targetValue) { + const difference = ((targetValue - baseValue) / baseValue) * 100; + const sign = difference >= 0 ? '+' : ''; + return `${sign}${difference.toFixed(2)}%`; +} + +if (process.argv.length !== 7) { + console.error('Usage: node compare-results.js baseCommitHash baseMetricsPath targetCommitHash targetMetricsPath outputFile'); + process.exit(1); +} + +const baseCommitHash = process.argv[2]; +const baseMetricsPath = process.argv[3]; +const targetCommitHash = process.argv[4]; +const targetMetricsPath = process.argv[5]; +const outputPath = process.argv[6]; + +generateComparisonTable(baseCommitHash, baseMetricsPath, targetCommitHash, targetMetricsPath, outputPath); diff --git a/k6/output/.gitkeep b/k6/output/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/k6/preload.js b/k6/preload.js new file mode 100644 index 000000000..5aed9907a --- /dev/null +++ b/k6/preload.js @@ -0,0 +1,10 @@ +import http from 'k6/http'; + +const BASE_URL = 'http://localhost:3001'; + +export default function preloadCache() { + const numberofTokens = http.get(`${BASE_URL}/tokens/count`); + http.get(`${BASE_URL}/tokens?size=${numberofTokens}`); + const numberofNodes = http.get(`${BASE_URL}/nodes/count`); + http.get(`${BASE_URL}/nodes?size=${numberofNodes}`); +} diff --git a/k6/script.js b/k6/script.js new file mode 100644 index 000000000..73a1e9d92 --- /dev/null +++ b/k6/script.js @@ -0,0 +1,99 @@ +import http from 'k6/http'; +import { sleep } from 'k6'; +import { Trend } from 'k6/metrics'; + +const BASE_URL = 'http://localhost:3001'; + +const accountsApiCallTrend = new Trend('accounts_http_req_duration', true); +const blocksApiCallTrend = new Trend('blocks_http_req_duration', true); +const mexPairsApiCallTrend = new Trend('mex_pairs_http_req_duration', true); +const mexTokensApiCallTrend = new Trend('mex_tokens_http_req_duration', true); +const mexFarmsApiCallTrend = new Trend('mex_farms_http_req_duration', true); +const nodesApiCallTrend = new Trend('nodes_http_req_duration', true); +const nodesAuctionsApiCallTrend = new Trend('nodes_auctions_http_req_duration', true); +const poolApiCallTrend = new Trend('pool_http_req_duration', true); +const tokensApiCallTrend = new Trend('tokens_http_req_duration', true); +const transactionsApiCallTrend = new Trend('transactions_http_req_duration', true); + + +function getScenarioDict(functionName) { + return { + executor: 'constant-vus', + vus: 10, + duration: '1m', + gracefulStop: '0s', + exec: functionName, + } +} + +export const options = { + scenarios: { + accounts: getScenarioDict('accounts'), + blocks: getScenarioDict('blocks'), + mexPairs: getScenarioDict('mexPairs'), + mexTokens: getScenarioDict('mexTokens'), + mexFarms: getScenarioDict('mexFarms'), + nodes: getScenarioDict('nodes'), + nodesAuctions: getScenarioDict('nodesAuctions'), + pool: getScenarioDict('pool'), + tokens: getScenarioDict('tokens'), + transactions: getScenarioDict('transactions'), + }, + discardResponseBodies: true, +}; + +export function accounts() { + const response = http.get(`${BASE_URL}/accounts`); + accountsApiCallTrend.add(response.timings.duration); +} + +export function blocks() { + const response = http.get(`${BASE_URL}/blocks`); + blocksApiCallTrend.add(response.timings.duration); +} + +export function mexPairs() { + const response = http.get(`${BASE_URL}/mex/pairs`); + mexPairsApiCallTrend.add(response.timings.duration); +} + +export function mexTokens() { + const response = http.get(`${BASE_URL}/mex/tokens`); + mexTokensApiCallTrend.add(response.timings.duration); +} + +export function mexFarms() { + const response = http.get(`${BASE_URL}/mex/farms`); + mexFarmsApiCallTrend.add(response.timings.duration); +} + +export function nodes() { + const response = http.get(`${BASE_URL}/nodes`); + nodesApiCallTrend.add(response.timings.duration); +} + +export function nodesAuctions() { + const response = http.get(`${BASE_URL}/nodes/auctions`); + nodesAuctionsApiCallTrend.add(response.timings.duration); +} + +export function pool() { + const response = http.get(`${BASE_URL}/pool`); + poolApiCallTrend.add(response.timings.duration); +} + +export function tokens() { + const response = http.get(`${BASE_URL}/tokens`); + tokensApiCallTrend.add(response.timings.duration); +} + +export function transactions() { + const response = http.get(`${BASE_URL}/transactions`); + transactionsApiCallTrend.add(response.timings.duration); +} + +export function handleSummary(data) { + return { + 'k6/output/summary.json': JSON.stringify(data), + }; +}