add comment

axolotl-ai-cloud · Aug 27, 2024 · 3b6a1ea · 3b6a1ea
1 parent 2075298
commit 3b6a1ea
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/axolotl/integrations/liger/__init__.py b/src/axolotl/integrations/liger/__init__.py
@@ -132,6 +132,8 @@ def pre_model_load(self, cfg):
  from .models.deepseekv2 import lce_forward as deepseekv2_lce_forward
 
  if cfg.liger_rope:
+ # The DeepseekV2 version of RoPE is different than upstream LLaMA.
+ # See https://github.com/linkedin/Liger-Kernel/issues/129#issuecomment-2313763528
  logging.warning("Fused liger_rope is not supported for DeepseekV2.")
  if cfg.liger_rms_norm:
  modeling_mod.DeepseekV2RMSNorm = LigerRMSNorm