Skip to content

Commit d09b048

Browse files
camchenrycamchenry
committed
ci: add profiling diff workflow
1 parent efae641 commit d09b048

File tree

1 file changed

+369
-0
lines changed

1 file changed

+369
-0
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 369 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,369 @@
1+
name: Benchmark PR
2+
3+
permissions:
4+
pull-requests: write
5+
contents: read
6+
7+
on:
8+
pull_request:
9+
types: [labeled, synchronize]
10+
11+
jobs:
12+
benchmark:
13+
if: (github.event.action == 'labeled' && github.event.label.name == 'bench' || github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'bench')) && github.event.pull_request.head.repo.full_name == github.repository
14+
runs-on: ubuntu-latest
15+
timeout-minutes: 60
16+
steps:
17+
- name: Checkout PR
18+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
19+
with:
20+
fetch-depth: 0
21+
submodules: true
22+
persist-credentials: false
23+
ref: ${{ github.event.pull_request.head.sha }}
24+
25+
- uses: oxc-project/setup-node@fdbf0dfd334c4e6d56ceeb77d91c76339c2a0885 # v1.0.4
26+
27+
- uses: ./.github/actions/setup
28+
29+
- name: Install required tools for benchmarking
30+
run: |
31+
sudo apt-get update
32+
sudo apt-get install -y \
33+
jq \
34+
valgrind \
35+
kcachegrind \
36+
graphviz
37+
38+
- name: Verify tool installations
39+
run: |
40+
echo "Tool versions:"
41+
valgrind --version
42+
callgrind_annotate --version || echo "callgrind_annotate is part of valgrind"
43+
cg_diff --version 2>&1 | head -1 || echo "cg_diff is part of valgrind"
44+
jq --version
45+
46+
- name: Clone vscode for benchmarking
47+
run: |
48+
mkdir -p benchmarks
49+
cd benchmarks
50+
# Clone vscode at pinned commit (v1.99.0 release)
51+
git clone --depth 1 --single-branch --branch 1.99.0 https://github.com/microsoft/vscode
52+
53+
- name: Setup vscode for benchmarking
54+
run: |
55+
cd benchmarks/vscode
56+
# Install minimal dependencies needed for TypeScript compilation
57+
npm install --ignore-scripts
58+
59+
- name: Build PR binary
60+
run: just build
61+
62+
- name: Generate headless payload for vscode
63+
run: |
64+
mkdir -p benchmark-results/pr
65+
# Generate payload with all TypeScript files in vscode/src (using absolute paths)
66+
find "$(pwd)/benchmarks/vscode/src" -type f \( -name "*.ts" -o -name "*.tsx" \) | \
67+
jq -R -s '
68+
split("\n")[:-1] |
69+
{
70+
version: 2,
71+
configs: [
72+
{
73+
file_paths: .,
74+
rules: [
75+
{ name: "await-thenable" },
76+
{ name: "no-array-delete" },
77+
{ name: "no-base-to-string" },
78+
{ name: "no-confusing-void-expression" },
79+
{ name: "no-duplicate-type-constituents" },
80+
{ name: "no-floating-promises" },
81+
{ name: "no-for-in-array" },
82+
{ name: "no-implied-eval" },
83+
{ name: "no-meaningless-void-operator" },
84+
{ name: "no-misused-promises" },
85+
{ name: "no-unnecessary-type-assertion" },
86+
{ name: "no-unsafe-argument" },
87+
{ name: "no-unsafe-assignment" },
88+
{ name: "no-unsafe-call" },
89+
{ name: "no-unsafe-member-access" },
90+
{ name: "no-unsafe-return" }
91+
]
92+
}
93+
]
94+
}
95+
' > benchmark-results/pr/payload.json
96+
echo "Generated payload with $(jq '.configs[0].file_paths | length' benchmark-results/pr/payload.json) files"
97+
98+
- name: Run PR benchmark with CPU profiling
99+
run: |
100+
./tsgolint headless -cpuprof benchmark-results/pr/cpu.prof < benchmark-results/pr/payload.json
101+
102+
- name: Generate PR profile summary
103+
run: |
104+
go tool pprof -top benchmark-results/pr/cpu.prof > benchmark-results/pr/profile.txt
105+
106+
- name: Run PR benchmark with Valgrind (Callgrind)
107+
run: |
108+
valgrind --tool=callgrind \
109+
--callgrind-out-file=benchmark-results/pr/callgrind.out \
110+
--cache-sim=yes \
111+
--branch-sim=yes \
112+
--simulate-hwpref=yes \
113+
--simulate-wb=yes \
114+
./tsgolint headless < benchmark-results/pr/payload.json
115+
116+
- name: Generate PR Valgrind summary
117+
run: |
118+
# Generate human-readable summary
119+
callgrind_annotate --auto=yes benchmark-results/pr/callgrind.out > benchmark-results/pr/callgrind.txt
120+
121+
- name: Checkout base branch
122+
run: |
123+
git fetch origin "${{ github.event.pull_request.base.ref }}"
124+
git checkout "origin/${{ github.event.pull_request.base.ref }}"
125+
git submodule update --init --recursive
126+
127+
- name: Apply typescript-go patches for base
128+
run: |
129+
pushd typescript-go
130+
git am --3way --no-gpg-sign ../patches/*.patch
131+
popd
132+
133+
- name: Expose typescript-go collections package for base
134+
run: |
135+
mkdir -p internal/collections
136+
find ./typescript-go/internal/collections -type f ! -name '*_test.go' -exec cp {} internal/collections/ \;
137+
138+
- name: Build base binary
139+
run: just build
140+
141+
- name: Generate headless payload for vscode (base)
142+
run: |
143+
mkdir -p benchmark-results/base
144+
# Generate payload with all TypeScript files in vscode/src (using absolute paths)
145+
find "$(pwd)/benchmarks/vscode/src" -type f \( -name "*.ts" -o -name "*.tsx" \) | \
146+
jq -R -s '
147+
split("\n")[:-1] |
148+
{
149+
version: 2,
150+
configs: [
151+
{
152+
file_paths: .,
153+
rules: [
154+
{ name: "await-thenable" },
155+
{ name: "no-array-delete" },
156+
{ name: "no-base-to-string" },
157+
{ name: "no-confusing-void-expression" },
158+
{ name: "no-duplicate-type-constituents" },
159+
{ name: "no-floating-promises" },
160+
{ name: "no-for-in-array" },
161+
{ name: "no-implied-eval" },
162+
{ name: "no-meaningless-void-operator" },
163+
{ name: "no-misused-promises" },
164+
{ name: "no-unnecessary-type-assertion" },
165+
{ name: "no-unsafe-argument" },
166+
{ name: "no-unsafe-assignment" },
167+
{ name: "no-unsafe-call" },
168+
{ name: "no-unsafe-member-access" },
169+
{ name: "no-unsafe-return" }
170+
]
171+
}
172+
]
173+
}
174+
' > benchmark-results/base/payload.json
175+
echo "Generated payload with $(jq '.configs[0].file_paths | length' benchmark-results/base/payload.json) files"
176+
177+
- name: Run base benchmark with CPU profiling
178+
run: |
179+
./tsgolint headless -cpuprof benchmark-results/base/cpu.prof < benchmark-results/base/payload.json
180+
181+
- name: Generate base profile summary
182+
run: |
183+
go tool pprof -top benchmark-results/base/cpu.prof > benchmark-results/base/profile.txt
184+
185+
- name: Run base benchmark with Valgrind (Callgrind)
186+
run: |
187+
valgrind --tool=callgrind \
188+
--callgrind-out-file=benchmark-results/base/callgrind.out \
189+
--cache-sim=yes \
190+
--branch-sim=yes \
191+
--simulate-hwpref=yes \
192+
--simulate-wb=yes \
193+
./tsgolint headless < benchmark-results/base/payload.json
194+
195+
- name: Generate base Valgrind summary
196+
run: |
197+
# Generate human-readable summary
198+
callgrind_annotate --auto=yes benchmark-results/base/callgrind.out > benchmark-results/base/callgrind.txt
199+
200+
- name: Compare profiles
201+
id: compare
202+
run: |
203+
BASE_REF="${{ github.event.pull_request.base.ref }}"
204+
HEAD_REF="${{ github.event.pull_request.head.ref }}"
205+
206+
echo "## Benchmark Results" > benchmark-results/comment.md
207+
echo "" >> benchmark-results/comment.md
208+
echo "**Benchmark Mode:** Headless ($(jq '.configs[0].file_paths | length' benchmark-results/pr/payload.json) files, $(jq '.configs[0].rules | length' benchmark-results/pr/payload.json) rules)" >> benchmark-results/comment.md
209+
echo "" >> benchmark-results/comment.md
210+
echo "### CPU Profile Comparison" >> benchmark-results/comment.md
211+
echo "" >> benchmark-results/comment.md
212+
echo "#### Base Branch ($BASE_REF)" >> benchmark-results/comment.md
213+
echo '```' >> benchmark-results/comment.md
214+
head -n 20 benchmark-results/base/profile.txt >> benchmark-results/comment.md
215+
echo '```' >> benchmark-results/comment.md
216+
echo "" >> benchmark-results/comment.md
217+
echo "#### PR Branch ($HEAD_REF)" >> benchmark-results/comment.md
218+
echo '```' >> benchmark-results/comment.md
219+
head -n 20 benchmark-results/pr/profile.txt >> benchmark-results/comment.md
220+
echo '```' >> benchmark-results/comment.md
221+
echo "" >> benchmark-results/comment.md
222+
echo "### Differential Profile" >> benchmark-results/comment.md
223+
echo "" >> benchmark-results/comment.md
224+
echo "Comparing PR against base (positive values = PR is slower):" >> benchmark-results/comment.md
225+
echo '```' >> benchmark-results/comment.md
226+
go tool pprof -top -base benchmark-results/base/cpu.prof benchmark-results/pr/cpu.prof | head -n 25 >> benchmark-results/comment.md || echo "No significant differences found" >> benchmark-results/comment.md
227+
echo '```' >> benchmark-results/comment.md
228+
echo "" >> benchmark-results/comment.md
229+
echo "### Valgrind (Callgrind) Performance Analysis" >> benchmark-results/comment.md
230+
echo "" >> benchmark-results/comment.md
231+
echo "CPU simulation with cache and branch prediction modeling:" >> benchmark-results/comment.md
232+
echo "" >> benchmark-results/comment.md
233+
echo "#### Base Branch ($BASE_REF)" >> benchmark-results/comment.md
234+
echo '```' >> benchmark-results/comment.md
235+
head -n 30 benchmark-results/base/callgrind.txt >> benchmark-results/comment.md
236+
echo '```' >> benchmark-results/comment.md
237+
echo "" >> benchmark-results/comment.md
238+
echo "#### PR Branch ($HEAD_REF)" >> benchmark-results/comment.md
239+
echo '```' >> benchmark-results/comment.md
240+
head -n 30 benchmark-results/pr/callgrind.txt >> benchmark-results/comment.md
241+
echo '```' >> benchmark-results/comment.md
242+
echo "" >> benchmark-results/comment.md
243+
echo "#### Instruction Count Comparison" >> benchmark-results/comment.md
244+
echo '```' >> benchmark-results/comment.md
245+
# Extract instruction counts for comparison
246+
BASE_IR=$(grep -E "^[ ]*[0-9,]+ +\*?PROGRAM TOTALS" benchmark-results/base/callgrind.txt | awk '{gsub(/,/,"",$1); print $1}')
247+
PR_IR=$(grep -E "^[ ]*[0-9,]+ +\*?PROGRAM TOTALS" benchmark-results/pr/callgrind.txt | awk '{gsub(/,/,"",$1); print $1}')
248+
if [ -n "$BASE_IR" ] && [ -n "$PR_IR" ]; then
249+
DIFF=$((PR_IR - BASE_IR))
250+
PERCENT=$(awk "BEGIN {printf \"%.2f\", ($DIFF / $BASE_IR) * 100}")
251+
echo "Base: $(printf "%'d" $BASE_IR) instructions" >> benchmark-results/comment.md
252+
echo "PR: $(printf "%'d" $PR_IR) instructions" >> benchmark-results/comment.md
253+
echo "Diff: $(printf "%'d" $DIFF) instructions ($PERCENT%)" >> benchmark-results/comment.md
254+
else
255+
echo "Unable to extract instruction counts" >> benchmark-results/comment.md
256+
fi
257+
echo '```' >> benchmark-results/comment.md
258+
echo "" >> benchmark-results/comment.md
259+
echo "<details>" >> benchmark-results/comment.md
260+
echo "<summary>View CPU Profiles</summary>" >> benchmark-results/comment.md
261+
echo "" >> benchmark-results/comment.md
262+
echo "**Note:** Download the benchmark artifacts to analyze CPU profiles locally with pprof." >> benchmark-results/comment.md
263+
echo "" >> benchmark-results/comment.md
264+
echo '```bash' >> benchmark-results/comment.md
265+
echo "# Interactive analysis" >> benchmark-results/comment.md
266+
echo "go tool pprof -http=:8080 -base base/cpu.prof pr/cpu.prof" >> benchmark-results/comment.md
267+
echo '```' >> benchmark-results/comment.md
268+
echo "" >> benchmark-results/comment.md
269+
echo "</details>" >> benchmark-results/comment.md
270+
271+
- name: Upload benchmark artifacts
272+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0
273+
with:
274+
name: benchmark-profiles
275+
path: benchmark-results/
276+
retention-days: 30
277+
278+
- name: Post benchmark results
279+
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
280+
with:
281+
script: |
282+
const fs = require('fs');
283+
const comment = fs.readFileSync('benchmark-results/comment.md', 'utf8');
284+
285+
const { data: comments } = await github.rest.issues.listComments({
286+
owner: context.repo.owner,
287+
repo: context.repo.repo,
288+
issue_number: context.issue.number,
289+
});
290+
291+
const botComment = comments.find(comment =>
292+
comment.user.type === 'Bot' &&
293+
comment.body.includes('## Benchmark Results')
294+
);
295+
296+
const commentBody = comment + '\n\n---\n\n📊 Download the [benchmark artifacts](' +
297+
`https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` +
298+
') to view flame graphs and detailed profiles.';
299+
300+
if (botComment) {
301+
await github.rest.issues.updateComment({
302+
owner: context.repo.owner,
303+
repo: context.repo.repo,
304+
comment_id: botComment.id,
305+
body: commentBody
306+
});
307+
} else {
308+
await github.rest.issues.createComment({
309+
owner: context.repo.owner,
310+
repo: context.repo.repo,
311+
issue_number: context.issue.number,
312+
body: commentBody
313+
});
314+
}
315+
316+
- name: Generate profile comparison report
317+
run: |
318+
echo "## Profile Analysis" > benchmark-results/analysis.md
319+
echo "" >> benchmark-results/analysis.md
320+
echo "### Instructions for Viewing Flame Graphs" >> benchmark-results/analysis.md
321+
echo "" >> benchmark-results/analysis.md
322+
echo "1. Download the benchmark-profiles artifact from this workflow run" >> benchmark-results/analysis.md
323+
echo "2. Extract the archive" >> benchmark-results/analysis.md
324+
echo "3. Open \`base/flamegraph.svg\` and \`pr/flamegraph.svg\` in a browser" >> benchmark-results/analysis.md
325+
echo "4. Compare the two flame graphs side by side" >> benchmark-results/analysis.md
326+
echo "" >> benchmark-results/analysis.md
327+
echo "### Using pprof for Interactive Analysis" >> benchmark-results/analysis.md
328+
echo "" >> benchmark-results/analysis.md
329+
echo '```bash' >> benchmark-results/analysis.md
330+
echo "# View differential profile interactively" >> benchmark-results/analysis.md
331+
echo "go tool pprof -http=:8080 -base base/cpu.prof pr/cpu.prof" >> benchmark-results/analysis.md
332+
echo "" >> benchmark-results/analysis.md
333+
echo "# Generate diff flame graph" >> benchmark-results/analysis.md
334+
echo "go tool pprof -svg -base base/cpu.prof pr/cpu.prof > diff.svg" >> benchmark-results/analysis.md
335+
echo '```' >> benchmark-results/analysis.md
336+
echo "" >> benchmark-results/analysis.md
337+
echo "### Using Valgrind/Callgrind for Detailed CPU Simulation" >> benchmark-results/analysis.md
338+
echo "" >> benchmark-results/analysis.md
339+
echo "Valgrind's Callgrind tool provides detailed instruction-level analysis with cache simulation:" >> benchmark-results/analysis.md
340+
echo "" >> benchmark-results/analysis.md
341+
echo '```bash' >> benchmark-results/analysis.md
342+
echo "# View callgrind output with KCachegrind (Linux/Mac)" >> benchmark-results/analysis.md
343+
echo "kcachegrind base/callgrind.out" >> benchmark-results/analysis.md
344+
echo "kcachegrind pr/callgrind.out" >> benchmark-results/analysis.md
345+
echo "" >> benchmark-results/analysis.md
346+
echo "# Or use QCachegrind on macOS" >> benchmark-results/analysis.md
347+
echo "qcachegrind base/callgrind.out" >> benchmark-results/analysis.md
348+
echo "" >> benchmark-results/analysis.md
349+
echo "# Compare callgrind outputs directly" >> benchmark-results/analysis.md
350+
echo "cg_diff base/callgrind.out pr/callgrind.out > diff.callgrind" >> benchmark-results/analysis.md
351+
echo "callgrind_annotate diff.callgrind" >> benchmark-results/analysis.md
352+
echo '```' >> benchmark-results/analysis.md
353+
echo "" >> benchmark-results/analysis.md
354+
echo "Callgrind provides:" >> benchmark-results/analysis.md
355+
echo "- Instruction count (deterministic, platform-independent)" >> benchmark-results/analysis.md
356+
echo "- Cache miss simulation (L1/L2/LL data/instruction caches)" >> benchmark-results/analysis.md
357+
echo "- Branch prediction simulation" >> benchmark-results/analysis.md
358+
echo "- Hardware prefetch simulation" >> benchmark-results/analysis.md
359+
echo "- Write-back cache simulation" >> benchmark-results/analysis.md
360+
echo '```' >> benchmark-results/analysis.md
361+
362+
cat benchmark-results/analysis.md
363+
364+
- name: Upload analysis instructions
365+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0
366+
with:
367+
name: analysis-instructions
368+
path: benchmark-results/analysis.md
369+
retention-days: 30

0 commit comments

Comments
 (0)