add kernel config tuning way to get better performance. (#681)

ModelTC · Dec 25, 2024 · 6a42960 · 6a42960
1 parent e3eea7d
commit 6a42960
Show file tree

Hide file tree

Showing 18 changed files with 2,297 additions and 260 deletions.
diff --git a/lightllm/common/all_kernel_configs/__init__.py b/lightllm/common/all_kernel_configs/__init__.py
diff --git a/lightllm/common/basemodel/layer_weights/meta_weights/fused_moe_weight.py b/lightllm/common/basemodel/layer_weights/meta_weights/fused_moe_weight.py
@@ -50,7 +50,10 @@ def experts(self, input_tensor, router_logits, top_k, renormalize, use_grouped_t
         w1, w1_scale = self.w1
         w2, w2_scale = self.w2
         use_fp8_w8a8 = self.quant_method is not None
-        fused_experts(
+
+        from lightllm.common.fused_moe.grouped_fused_moe import fused_experts_impl
+
+        fused_experts_impl(
             hidden_states=input_tensor,
             w1=w1,
             w2=w2,
@@ -61,6 +64,7 @@ def experts(self, input_tensor, router_logits, top_k, renormalize, use_grouped_t
             w1_scale=w1_scale,
             w2_scale=w2_scale,
         )
+        return
 
     def _fuse(self):
         with self.lock: