Skip to content

Commit ff790ca

Browse files
camchenrycamchenry
committed
ci: add profiling diff workflow
1 parent 36f5ab9 commit ff790ca

File tree

1 file changed

+268
-0
lines changed

1 file changed

+268
-0
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
name: Benchmark PR
2+
3+
permissions:
4+
pull-requests: write
5+
contents: read
6+
7+
on:
8+
pull_request:
9+
types: [labeled]
10+
11+
jobs:
12+
benchmark:
13+
if: github.event.label.name == 'bench'
14+
runs-on: ubuntu-latest
15+
timeout-minutes: 60
16+
steps:
17+
- name: Checkout PR
18+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
19+
with:
20+
fetch-depth: 0
21+
submodules: true
22+
persist-credentials: false
23+
ref: ${{ github.event.pull_request.head.sha }}
24+
25+
- uses: oxc-project/setup-node@fdbf0dfd334c4e6d56ceeb77d91c76339c2a0885 # v1.0.4
26+
27+
- uses: ./.github/actions/setup
28+
29+
- name: Install jq for payload generation
30+
run: sudo apt-get update && sudo apt-get install -y jq
31+
32+
- name: Clone vscode for benchmarking
33+
run: |
34+
mkdir -p benchmarks
35+
cd benchmarks
36+
# Clone vscode at pinned commit (v1.99.0 release)
37+
git clone --depth 1 --single-branch --branch 1.99.0 https://github.com/microsoft/vscode
38+
39+
- name: Setup vscode for benchmarking
40+
run: |
41+
cd benchmarks/vscode
42+
# Install minimal dependencies needed for TypeScript compilation
43+
npm install --ignore-scripts
44+
45+
- name: Build PR binary
46+
run: just build
47+
48+
- name: Generate headless payload for vscode
49+
run: |
50+
mkdir -p benchmark-results/pr
51+
# Generate payload with all TypeScript files in vscode/src
52+
find benchmarks/vscode/src -type f \( -name "*.ts" -o -name "*.tsx" \) | \
53+
jq -R -s '
54+
split("\n")[:-1] |
55+
{
56+
version: 2,
57+
configs: [
58+
{
59+
file_paths: .,
60+
rules: [
61+
{ name: "await-thenable" },
62+
{ name: "no-array-delete" },
63+
{ name: "no-base-to-string" },
64+
{ name: "no-confusing-void-expression" },
65+
{ name: "no-duplicate-type-constituents" },
66+
{ name: "no-floating-promises" },
67+
{ name: "no-for-in-array" },
68+
{ name: "no-implied-eval" },
69+
{ name: "no-meaningless-void-operator" },
70+
{ name: "no-misused-promises" },
71+
{ name: "no-unnecessary-type-assertion" },
72+
{ name: "no-unsafe-argument" },
73+
{ name: "no-unsafe-assignment" },
74+
{ name: "no-unsafe-call" },
75+
{ name: "no-unsafe-member-access" },
76+
{ name: "no-unsafe-return" }
77+
]
78+
}
79+
]
80+
}
81+
' > benchmark-results/pr/payload.json
82+
echo "Generated payload with $(jq '.configs[0].file_paths | length' benchmark-results/pr/payload.json) files"
83+
84+
- name: Run PR benchmark with CPU profiling
85+
run: |
86+
./tsgolint headless --cpuprof benchmark-results/pr/cpu.prof < benchmark-results/pr/payload.json
87+
88+
- name: Generate PR profile summary
89+
run: |
90+
go tool pprof -top benchmark-results/pr/cpu.prof > benchmark-results/pr/profile.txt
91+
92+
- name: Checkout base branch
93+
run: |
94+
git fetch origin ${{ github.event.pull_request.base.ref }}
95+
git checkout origin/${{ github.event.pull_request.base.ref }}
96+
git submodule update --init --recursive
97+
98+
- name: Apply typescript-go patches for base
99+
run: |
100+
pushd typescript-go
101+
git am --3way --no-gpg-sign ../patches/*.patch
102+
popd
103+
104+
- name: Expose typescript-go collections package for base
105+
run: |
106+
mkdir -p internal/collections
107+
find ./typescript-go/internal/collections -type f ! -name '*_test.go' -exec cp {} internal/collections/ \;
108+
109+
- name: Build base binary
110+
run: just build
111+
112+
- name: Generate headless payload for vscode (base)
113+
run: |
114+
mkdir -p benchmark-results/base
115+
# Generate payload with all TypeScript files in vscode/src
116+
find benchmarks/vscode/src -type f \( -name "*.ts" -o -name "*.tsx" \) | \
117+
jq -R -s '
118+
split("\n")[:-1] |
119+
{
120+
version: 2,
121+
configs: [
122+
{
123+
tsconfig: "benchmarks/vscode/src/tsconfig.json",
124+
file_paths: .,
125+
rules: [
126+
{ name: "await-thenable" },
127+
{ name: "no-array-delete" },
128+
{ name: "no-base-to-string" },
129+
{ name: "no-confusing-void-expression" },
130+
{ name: "no-duplicate-type-constituents" },
131+
{ name: "no-floating-promises" },
132+
{ name: "no-for-in-array" },
133+
{ name: "no-implied-eval" },
134+
{ name: "no-meaningless-void-operator" },
135+
{ name: "no-misused-promises" },
136+
{ name: "no-unnecessary-type-assertion" },
137+
{ name: "no-unsafe-argument" },
138+
{ name: "no-unsafe-assignment" },
139+
{ name: "no-unsafe-call" },
140+
{ name: "no-unsafe-member-access" },
141+
{ name: "no-unsafe-return" }
142+
]
143+
}
144+
]
145+
}
146+
' > benchmark-results/base/payload.json
147+
echo "Generated payload with $(jq '.configs[0].file_paths | length' benchmark-results/base/payload.json) files"
148+
149+
- name: Run base benchmark with CPU profiling
150+
run: |
151+
./tsgolint headless --cpuprof benchmark-results/base/cpu.prof < benchmark-results/base/payload.json
152+
153+
- name: Generate base profile summary
154+
run: |
155+
go tool pprof -top benchmark-results/base/cpu.prof > benchmark-results/base/profile.txt
156+
157+
- name: Compare profiles
158+
id: compare
159+
run: |
160+
echo "## Benchmark Results" > benchmark-results/comment.md
161+
echo "" >> benchmark-results/comment.md
162+
echo "**Benchmark Mode:** Headless ($(jq '.configs[0].file_paths | length' benchmark-results/pr/payload.json) files, $(jq '.configs[0].rules | length' benchmark-results/pr/payload.json) rules)" >> benchmark-results/comment.md
163+
echo "" >> benchmark-results/comment.md
164+
echo "### CPU Profile Comparison" >> benchmark-results/comment.md
165+
echo "" >> benchmark-results/comment.md
166+
echo "#### Base Branch (${{ github.event.pull_request.base.ref }})" >> benchmark-results/comment.md
167+
echo '```' >> benchmark-results/comment.md
168+
head -n 20 benchmark-results/base/profile.txt >> benchmark-results/comment.md
169+
echo '```' >> benchmark-results/comment.md
170+
echo "" >> benchmark-results/comment.md
171+
echo "#### PR Branch (${{ github.event.pull_request.head.ref }})" >> benchmark-results/comment.md
172+
echo '```' >> benchmark-results/comment.md
173+
head -n 20 benchmark-results/pr/profile.txt >> benchmark-results/comment.md
174+
echo '```' >> benchmark-results/comment.md
175+
echo "" >> benchmark-results/comment.md
176+
echo "### Differential Profile" >> benchmark-results/comment.md
177+
echo "" >> benchmark-results/comment.md
178+
echo "Comparing PR against base (positive values = PR is slower):" >> benchmark-results/comment.md
179+
echo '```' >> benchmark-results/comment.md
180+
go tool pprof -top -base benchmark-results/base/cpu.prof benchmark-results/pr/cpu.prof | head -n 25 >> benchmark-results/comment.md || echo "No significant differences found" >> benchmark-results/comment.md
181+
echo '```' >> benchmark-results/comment.md
182+
echo "" >> benchmark-results/comment.md
183+
echo "<details>" >> benchmark-results/comment.md
184+
echo "<summary>View CPU Profiles</summary>" >> benchmark-results/comment.md
185+
echo "" >> benchmark-results/comment.md
186+
echo "**Note:** Download the benchmark artifacts to analyze CPU profiles locally with pprof." >> benchmark-results/comment.md
187+
echo "" >> benchmark-results/comment.md
188+
echo '```bash' >> benchmark-results/comment.md
189+
echo "# Interactive analysis" >> benchmark-results/comment.md
190+
echo "go tool pprof -http=:8080 -base base/cpu.prof pr/cpu.prof" >> benchmark-results/comment.md
191+
echo '```' >> benchmark-results/comment.md
192+
echo "" >> benchmark-results/comment.md
193+
echo "</details>" >> benchmark-results/comment.md
194+
195+
- name: Upload benchmark artifacts
196+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0
197+
with:
198+
name: benchmark-profiles
199+
path: benchmark-results/
200+
retention-days: 30
201+
202+
- name: Post benchmark results
203+
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
204+
with:
205+
script: |
206+
const fs = require('fs');
207+
const comment = fs.readFileSync('benchmark-results/comment.md', 'utf8');
208+
209+
const { data: comments } = await github.rest.issues.listComments({
210+
owner: context.repo.owner,
211+
repo: context.repo.repo,
212+
issue_number: context.issue.number,
213+
});
214+
215+
const botComment = comments.find(comment =>
216+
comment.user.type === 'Bot' &&
217+
comment.body.includes('## Benchmark Results')
218+
);
219+
220+
const commentBody = comment + '\n\n---\n\n📊 Download the [benchmark artifacts](' +
221+
`https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` +
222+
') to view flame graphs and detailed profiles.';
223+
224+
if (botComment) {
225+
await github.rest.issues.updateComment({
226+
owner: context.repo.owner,
227+
repo: context.repo.repo,
228+
comment_id: botComment.id,
229+
body: commentBody
230+
});
231+
} else {
232+
await github.rest.issues.createComment({
233+
owner: context.repo.owner,
234+
repo: context.repo.repo,
235+
issue_number: context.issue.number,
236+
body: commentBody
237+
});
238+
}
239+
240+
- name: Generate profile comparison report
241+
run: |
242+
echo "## Profile Analysis" > benchmark-results/analysis.md
243+
echo "" >> benchmark-results/analysis.md
244+
echo "### Instructions for Viewing Flame Graphs" >> benchmark-results/analysis.md
245+
echo "" >> benchmark-results/analysis.md
246+
echo "1. Download the benchmark-profiles artifact from this workflow run" >> benchmark-results/analysis.md
247+
echo "2. Extract the archive" >> benchmark-results/analysis.md
248+
echo "3. Open \`base/flamegraph.svg\` and \`pr/flamegraph.svg\` in a browser" >> benchmark-results/analysis.md
249+
echo "4. Compare the two flame graphs side by side" >> benchmark-results/analysis.md
250+
echo "" >> benchmark-results/analysis.md
251+
echo "### Using pprof for Interactive Analysis" >> benchmark-results/analysis.md
252+
echo "" >> benchmark-results/analysis.md
253+
echo '```bash' >> benchmark-results/analysis.md
254+
echo "# View differential profile interactively" >> benchmark-results/analysis.md
255+
echo "go tool pprof -http=:8080 -base base/cpu.prof pr/cpu.prof" >> benchmark-results/analysis.md
256+
echo "" >> benchmark-results/analysis.md
257+
echo "# Generate diff flame graph" >> benchmark-results/analysis.md
258+
echo "go tool pprof -svg -base base/cpu.prof pr/cpu.prof > diff.svg" >> benchmark-results/analysis.md
259+
echo '```' >> benchmark-results/analysis.md
260+
261+
cat benchmark-results/analysis.md
262+
263+
- name: Upload analysis instructions
264+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0
265+
with:
266+
name: analysis-instructions
267+
path: benchmark-results/analysis.md
268+
retention-days: 30

0 commit comments

Comments
 (0)