Skip to content

Commit e4ee658

Browse files
tomeras91tomeras91
authored
[Model] add optimal triton fused moe configs for NemotronH MoE (#27967)
Signed-off-by: Tomer Asida <57313761+tomeras91@users.noreply.github.com>
1 parent 77f8001 commit e4ee658

File tree

5 files changed

+589
-0
lines changed

5 files changed

+589
-0
lines changed

‎benchmarks/kernels/benchmark_moe.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,7 @@ def main(args: argparse.Namespace):
590590
"DeepseekV3ForCausalLM",
591591
"DeepseekV32ForCausalLM",
592592
"Glm4MoeForCausalLM",
593+
"NemotronHForCausalLM",
593594
):
594595
E = config.n_routed_experts
595596
topk = config.num_experts_per_tok

‎vllm/model_executor/layers/fused_moe/configs/E=128,N=1856,device_name=NVIDIA_H100_80GB_HBM3.json‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
{
2+
"triton_version": "3.5.0",
3+
"1": {
4+
"BLOCK_SIZE_M": 16,
5+
"BLOCK_SIZE_N": 32,
6+
"BLOCK_SIZE_K": 64,
7+
"GROUP_SIZE_M": 1,
8+
"num_warps": 4,
9+
"num_stages": 4
10+
},
11+
"2": {
12+
"BLOCK_SIZE_M": 16,
13+
"BLOCK_SIZE_N": 32,
14+
"BLOCK_SIZE_K": 64,
15+
"GROUP_SIZE_M": 1,
16+
"num_warps": 4,
17+
"num_stages": 3
18+
},
19+
"4": {
20+
"BLOCK_SIZE_M": 16,
21+
"BLOCK_SIZE_N": 64,
22+
"BLOCK_SIZE_K": 128,
23+
"GROUP_SIZE_M": 1,
24+
"num_warps": 8,
25+
"num_stages": 4
26+
},
27+
"8": {
28+
"BLOCK_SIZE_M": 16,
29+
"BLOCK_SIZE_N": 64,
30+
"BLOCK_SIZE_K": 128,
31+
"GROUP_SIZE_M": 64,
32+
"num_warps": 4,
33+
"num_stages": 5
34+
},
35+
"16": {
36+
"BLOCK_SIZE_M": 16,
37+
"BLOCK_SIZE_N": 128,
38+
"BLOCK_SIZE_K": 128,
39+
"GROUP_SIZE_M": 1,
40+
"num_warps": 8,
41+
"num_stages": 5
42+
},
43+
"24": {
44+
"BLOCK_SIZE_M": 16,
45+
"BLOCK_SIZE_N": 32,
46+
"BLOCK_SIZE_K": 256,
47+
"GROUP_SIZE_M": 64,
48+
"num_warps": 4,
49+
"num_stages": 3
50+
},
51+
"32": {
52+
"BLOCK_SIZE_M": 16,
53+
"BLOCK_SIZE_N": 32,
54+
"BLOCK_SIZE_K": 256,
55+
"GROUP_SIZE_M": 1,
56+
"num_warps": 4,
57+
"num_stages": 5
58+
},
59+
"48": {
60+
"BLOCK_SIZE_M": 16,
61+
"BLOCK_SIZE_N": 64,
62+
"BLOCK_SIZE_K": 256,
63+
"GROUP_SIZE_M": 1,
64+
"num_warps": 4,
65+
"num_stages": 5
66+
},
67+
"64": {
68+
"BLOCK_SIZE_M": 16,
69+
"BLOCK_SIZE_N": 32,
70+
"BLOCK_SIZE_K": 256,
71+
"GROUP_SIZE_M": 1,
72+
"num_warps": 4,
73+
"num_stages": 5
74+
},
75+
"96": {
76+
"BLOCK_SIZE_M": 16,
77+
"BLOCK_SIZE_N": 32,
78+
"BLOCK_SIZE_K": 256,
79+
"GROUP_SIZE_M": 1,
80+
"num_warps": 4,
81+
"num_stages": 5
82+
},
83+
"128": {
84+
"BLOCK_SIZE_M": 16,
85+
"BLOCK_SIZE_N": 32,
86+
"BLOCK_SIZE_K": 256,
87+
"GROUP_SIZE_M": 1,
88+
"num_warps": 4,
89+
"num_stages": 5
90+
},
91+
"256": {
92+
"BLOCK_SIZE_M": 32,
93+
"BLOCK_SIZE_N": 32,
94+
"BLOCK_SIZE_K": 128,
95+
"GROUP_SIZE_M": 1,
96+
"num_warps": 4,
97+
"num_stages": 5
98+
},
99+
"512": {
100+
"BLOCK_SIZE_M": 64,
101+
"BLOCK_SIZE_N": 128,
102+
"BLOCK_SIZE_K": 128,
103+
"GROUP_SIZE_M": 1,
104+
"num_warps": 8,
105+
"num_stages": 4
106+
},
107+
"1024": {
108+
"BLOCK_SIZE_M": 64,
109+
"BLOCK_SIZE_N": 128,
110+
"BLOCK_SIZE_K": 128,
111+
"GROUP_SIZE_M": 1,
112+
"num_warps": 8,
113+
"num_stages": 4
114+
},
115+
"1536": {
116+
"BLOCK_SIZE_M": 128,
117+
"BLOCK_SIZE_N": 128,
118+
"BLOCK_SIZE_K": 64,
119+
"GROUP_SIZE_M": 1,
120+
"num_warps": 4,
121+
"num_stages": 3
122+
},
123+
"2048": {
124+
"BLOCK_SIZE_M": 128,
125+
"BLOCK_SIZE_N": 128,
126+
"BLOCK_SIZE_K": 64,
127+
"GROUP_SIZE_M": 1,
128+
"num_warps": 8,
129+
"num_stages": 5
130+
},
131+
"3072": {
132+
"BLOCK_SIZE_M": 128,
133+
"BLOCK_SIZE_N": 256,
134+
"BLOCK_SIZE_K": 64,
135+
"GROUP_SIZE_M": 1,
136+
"num_warps": 8,
137+
"num_stages": 4
138+
},
139+
"4096": {
140+
"BLOCK_SIZE_M": 128,
141+
"BLOCK_SIZE_N": 256,
142+
"BLOCK_SIZE_K": 64,
143+
"GROUP_SIZE_M": 1,
144+
"num_warps": 8,
145+
"num_stages": 4
146+
}
147+
}

‎vllm/model_executor/layers/fused_moe/configs/E=128,N=1856,device_name=NVIDIA_L40S.json‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
{
2+
"triton_version": "3.5.0",
3+
"1": {
4+
"BLOCK_SIZE_M": 16,
5+
"BLOCK_SIZE_N": 32,
6+
"BLOCK_SIZE_K": 256,
7+
"GROUP_SIZE_M": 1,
8+
"num_warps": 8,
9+
"num_stages": 4
10+
},
11+
"2": {
12+
"BLOCK_SIZE_M": 16,
13+
"BLOCK_SIZE_N": 32,
14+
"BLOCK_SIZE_K": 256,
15+
"GROUP_SIZE_M": 32,
16+
"num_warps": 4,
17+
"num_stages": 4
18+
},
19+
"4": {
20+
"BLOCK_SIZE_M": 32,
21+
"BLOCK_SIZE_N": 32,
22+
"BLOCK_SIZE_K": 128,
23+
"GROUP_SIZE_M": 16,
24+
"num_warps": 8,
25+
"num_stages": 5
26+
},
27+
"8": {
28+
"BLOCK_SIZE_M": 16,
29+
"BLOCK_SIZE_N": 32,
30+
"BLOCK_SIZE_K": 256,
31+
"GROUP_SIZE_M": 64,
32+
"num_warps": 4,
33+
"num_stages": 4
34+
},
35+
"16": {
36+
"BLOCK_SIZE_M": 32,
37+
"BLOCK_SIZE_N": 64,
38+
"BLOCK_SIZE_K": 256,
39+
"GROUP_SIZE_M": 16,
40+
"num_warps": 4,
41+
"num_stages": 3
42+
},
43+
"24": {
44+
"BLOCK_SIZE_M": 32,
45+
"BLOCK_SIZE_N": 32,
46+
"BLOCK_SIZE_K": 128,
47+
"GROUP_SIZE_M": 1,
48+
"num_warps": 8,
49+
"num_stages": 5
50+
},
51+
"32": {
52+
"BLOCK_SIZE_M": 32,
53+
"BLOCK_SIZE_N": 32,
54+
"BLOCK_SIZE_K": 128,
55+
"GROUP_SIZE_M": 1,
56+
"num_warps": 4,
57+
"num_stages": 5
58+
},
59+
"48": {
60+
"BLOCK_SIZE_M": 32,
61+
"BLOCK_SIZE_N": 32,
62+
"BLOCK_SIZE_K": 256,
63+
"GROUP_SIZE_M": 16,
64+
"num_warps": 4,
65+
"num_stages": 4
66+
},
67+
"64": {
68+
"BLOCK_SIZE_M": 16,
69+
"BLOCK_SIZE_N": 32,
70+
"BLOCK_SIZE_K": 256,
71+
"GROUP_SIZE_M": 32,
72+
"num_warps": 4,
73+
"num_stages": 5
74+
},
75+
"96": {
76+
"BLOCK_SIZE_M": 16,
77+
"BLOCK_SIZE_N": 32,
78+
"BLOCK_SIZE_K": 256,
79+
"GROUP_SIZE_M": 16,
80+
"num_warps": 8,
81+
"num_stages": 5
82+
},
83+
"128": {
84+
"BLOCK_SIZE_M": 16,
85+
"BLOCK_SIZE_N": 32,
86+
"BLOCK_SIZE_K": 256,
87+
"GROUP_SIZE_M": 1,
88+
"num_warps": 8,
89+
"num_stages": 5
90+
},
91+
"256": {
92+
"BLOCK_SIZE_M": 32,
93+
"BLOCK_SIZE_N": 32,
94+
"BLOCK_SIZE_K": 256,
95+
"GROUP_SIZE_M": 1,
96+
"num_warps": 4,
97+
"num_stages": 4
98+
},
99+
"512": {
100+
"BLOCK_SIZE_M": 32,
101+
"BLOCK_SIZE_N": 32,
102+
"BLOCK_SIZE_K": 256,
103+
"GROUP_SIZE_M": 64,
104+
"num_warps": 4,
105+
"num_stages": 4
106+
},
107+
"1024": {
108+
"BLOCK_SIZE_M": 64,
109+
"BLOCK_SIZE_N": 32,
110+
"BLOCK_SIZE_K": 256,
111+
"GROUP_SIZE_M": 64,
112+
"num_warps": 4,
113+
"num_stages": 3
114+
},
115+
"1536": {
116+
"BLOCK_SIZE_M": 64,
117+
"BLOCK_SIZE_N": 32,
118+
"BLOCK_SIZE_K": 128,
119+
"GROUP_SIZE_M": 64,
120+
"num_warps": 8,
121+
"num_stages": 5
122+
},
123+
"2048": {
124+
"BLOCK_SIZE_M": 64,
125+
"BLOCK_SIZE_N": 128,
126+
"BLOCK_SIZE_K": 256,
127+
"GROUP_SIZE_M": 64,
128+
"num_warps": 8,
129+
"num_stages": 2
130+
},
131+
"3072": {
132+
"BLOCK_SIZE_M": 64,
133+
"BLOCK_SIZE_N": 128,
134+
"BLOCK_SIZE_K": 128,
135+
"GROUP_SIZE_M": 1,
136+
"num_warps": 4,
137+
"num_stages": 3
138+
},
139+
"4096": {
140+
"BLOCK_SIZE_M": 128,
141+
"BLOCK_SIZE_N": 128,
142+
"BLOCK_SIZE_K": 128,
143+
"GROUP_SIZE_M": 32,
144+
"num_warps": 8,
145+
"num_stages": 2
146+
}
147+
}

0 commit comments

Comments
 (0)