Skip to content

Commit c2cc618

Browse files
camchenrycamchenry
committed
ci: add profiling diff workflow
1 parent 36f5ab9 commit c2cc618

File tree

1 file changed

+196
-0
lines changed

1 file changed

+196
-0
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
name: Benchmark PR
2+
3+
permissions:
4+
pull-requests: write
5+
contents: read
6+
7+
on:
8+
pull_request:
9+
types: [labeled]
10+
11+
jobs:
12+
benchmark:
13+
if: github.event.label.name == 'bench'
14+
runs-on: ubuntu-latest
15+
timeout-minutes: 60
16+
steps:
17+
- name: Checkout PR
18+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
19+
with:
20+
fetch-depth: 0
21+
submodules: true
22+
persist-credentials: false
23+
ref: ${{ github.event.pull_request.head.sha }}
24+
25+
- uses: oxc-project/setup-node@fdbf0dfd334c4e6d56ceeb77d91c76339c2a0885 # v1.0.4
26+
27+
- uses: ./.github/actions/setup
28+
29+
- name: Install graphviz for flame graphs
30+
run: sudo apt-get update && sudo apt-get install -y graphviz
31+
32+
- name: Clone benchmark projects
33+
run: |
34+
cd benchmarks
35+
./clone-projects.sh
36+
37+
- name: Setup benchmark projects
38+
run: |
39+
cd benchmarks
40+
./setup.sh
41+
42+
- name: Build PR binary
43+
run: just build
44+
45+
- name: Run PR benchmark with CPU profiling
46+
run: |
47+
mkdir -p benchmark-results/pr
48+
export TSGOLINT_BENCHMARK_PROJECT=vscode
49+
cd benchmarks/vscode
50+
../../tsgolint --cpuprof ../../benchmark-results/pr/cpu.prof --tsconfig ./src/tsconfig.json
51+
continue-on-error: true
52+
53+
- name: Generate PR flame graph
54+
run: |
55+
go tool pprof -svg benchmark-results/pr/cpu.prof > benchmark-results/pr/flamegraph.svg
56+
go tool pprof -top benchmark-results/pr/cpu.prof > benchmark-results/pr/profile.txt
57+
58+
- name: Checkout base branch
59+
run: |
60+
git fetch origin ${{ github.event.pull_request.base.ref }}
61+
git checkout origin/${{ github.event.pull_request.base.ref }}
62+
git submodule update --init --recursive
63+
64+
- name: Apply typescript-go patches for base
65+
run: |
66+
pushd typescript-go
67+
git am --3way --no-gpg-sign ../patches/*.patch
68+
popd
69+
70+
- name: Expose typescript-go collections package for base
71+
run: |
72+
mkdir -p internal/collections
73+
find ./typescript-go/internal/collections -type f ! -name '*_test.go' -exec cp {} internal/collections/ \;
74+
75+
- name: Build base binary
76+
run: just build
77+
78+
- name: Run base benchmark with CPU profiling
79+
run: |
80+
mkdir -p benchmark-results/base
81+
export TSGOLINT_BENCHMARK_PROJECT=vscode
82+
cd benchmarks/vscode
83+
../../tsgolint --cpuprof ../../benchmark-results/base/cpu.prof --tsconfig ./src/tsconfig.json
84+
continue-on-error: true
85+
86+
- name: Generate base flame graph
87+
run: |
88+
go tool pprof -svg benchmark-results/base/cpu.prof > benchmark-results/base/flamegraph.svg
89+
go tool pprof -top benchmark-results/base/cpu.prof > benchmark-results/base/profile.txt
90+
91+
- name: Compare profiles
92+
id: compare
93+
run: |
94+
echo "## Benchmark Results" > benchmark-results/comment.md
95+
echo "" >> benchmark-results/comment.md
96+
echo "### CPU Profile Comparison" >> benchmark-results/comment.md
97+
echo "" >> benchmark-results/comment.md
98+
echo "#### Base Branch (${{ github.event.pull_request.base.ref }})" >> benchmark-results/comment.md
99+
echo '```' >> benchmark-results/comment.md
100+
head -n 20 benchmark-results/base/profile.txt >> benchmark-results/comment.md
101+
echo '```' >> benchmark-results/comment.md
102+
echo "" >> benchmark-results/comment.md
103+
echo "#### PR Branch (${{ github.event.pull_request.head.ref }})" >> benchmark-results/comment.md
104+
echo '```' >> benchmark-results/comment.md
105+
head -n 20 benchmark-results/pr/profile.txt >> benchmark-results/comment.md
106+
echo '```' >> benchmark-results/comment.md
107+
echo "" >> benchmark-results/comment.md
108+
echo "### Differential Profile" >> benchmark-results/comment.md
109+
echo "" >> benchmark-results/comment.md
110+
echo "Comparing PR against base (positive values = PR is slower):" >> benchmark-results/comment.md
111+
echo '```' >> benchmark-results/comment.md
112+
go tool pprof -top -base benchmark-results/base/cpu.prof benchmark-results/pr/cpu.prof | head -n 25 >> benchmark-results/comment.md || echo "No significant differences found" >> benchmark-results/comment.md
113+
echo '```' >> benchmark-results/comment.md
114+
echo "" >> benchmark-results/comment.md
115+
echo "<details>" >> benchmark-results/comment.md
116+
echo "<summary>View Full Flame Graphs</summary>" >> benchmark-results/comment.md
117+
echo "" >> benchmark-results/comment.md
118+
echo "**Note:** Flame graphs have been generated but GitHub doesn't render SVGs in comments." >> benchmark-results/comment.md
119+
echo "Download the artifacts to view them locally." >> benchmark-results/comment.md
120+
echo "" >> benchmark-results/comment.md
121+
echo "</details>" >> benchmark-results/comment.md
122+
123+
- name: Upload benchmark artifacts
124+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0
125+
with:
126+
name: benchmark-profiles
127+
path: benchmark-results/
128+
retention-days: 30
129+
130+
- name: Post benchmark results
131+
uses: actions/github-script@v7
132+
with:
133+
script: |
134+
const fs = require('fs');
135+
const comment = fs.readFileSync('benchmark-results/comment.md', 'utf8');
136+
137+
const { data: comments } = await github.rest.issues.listComments({
138+
owner: context.repo.owner,
139+
repo: context.repo.repo,
140+
issue_number: context.issue.number,
141+
});
142+
143+
const botComment = comments.find(comment =>
144+
comment.user.type === 'Bot' &&
145+
comment.body.includes('## Benchmark Results')
146+
);
147+
148+
const commentBody = comment + '\n\n---\n\n📊 Download the [benchmark artifacts](' +
149+
`https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` +
150+
') to view flame graphs and detailed profiles.';
151+
152+
if (botComment) {
153+
await github.rest.issues.updateComment({
154+
owner: context.repo.owner,
155+
repo: context.repo.repo,
156+
comment_id: botComment.id,
157+
body: commentBody
158+
});
159+
} else {
160+
await github.rest.issues.createComment({
161+
owner: context.repo.owner,
162+
repo: context.repo.repo,
163+
issue_number: context.issue.number,
164+
body: commentBody
165+
});
166+
}
167+
168+
- name: Generate profile comparison report
169+
run: |
170+
echo "## Profile Analysis" > benchmark-results/analysis.md
171+
echo "" >> benchmark-results/analysis.md
172+
echo "### Instructions for Viewing Flame Graphs" >> benchmark-results/analysis.md
173+
echo "" >> benchmark-results/analysis.md
174+
echo "1. Download the benchmark-profiles artifact from this workflow run" >> benchmark-results/analysis.md
175+
echo "2. Extract the archive" >> benchmark-results/analysis.md
176+
echo "3. Open \`base/flamegraph.svg\` and \`pr/flamegraph.svg\` in a browser" >> benchmark-results/analysis.md
177+
echo "4. Compare the two flame graphs side by side" >> benchmark-results/analysis.md
178+
echo "" >> benchmark-results/analysis.md
179+
echo "### Using pprof for Interactive Analysis" >> benchmark-results/analysis.md
180+
echo "" >> benchmark-results/analysis.md
181+
echo '```bash' >> benchmark-results/analysis.md
182+
echo "# View differential profile interactively" >> benchmark-results/analysis.md
183+
echo "go tool pprof -http=:8080 -base base/cpu.prof pr/cpu.prof" >> benchmark-results/analysis.md
184+
echo "" >> benchmark-results/analysis.md
185+
echo "# Generate diff flame graph" >> benchmark-results/analysis.md
186+
echo "go tool pprof -svg -base base/cpu.prof pr/cpu.prof > diff.svg" >> benchmark-results/analysis.md
187+
echo '```' >> benchmark-results/analysis.md
188+
189+
cat benchmark-results/analysis.md
190+
191+
- name: Upload analysis instructions
192+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0
193+
with:
194+
name: analysis-instructions
195+
path: benchmark-results/analysis.md
196+
retention-days: 30

0 commit comments

Comments
 (0)