From 3b6a1ea718ace2b9ef395738d27a20e1fe68aa33 Mon Sep 17 00:00:00 2001 From: Aman Karmani Date: Tue, 27 Aug 2024 23:49:46 +0000 Subject: [PATCH] add comment --- src/axolotl/integrations/liger/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/axolotl/integrations/liger/__init__.py b/src/axolotl/integrations/liger/__init__.py index 13456ea1c..b5c55de9c 100644 --- a/src/axolotl/integrations/liger/__init__.py +++ b/src/axolotl/integrations/liger/__init__.py @@ -132,6 +132,8 @@ def pre_model_load(self, cfg): from .models.deepseekv2 import lce_forward as deepseekv2_lce_forward if cfg.liger_rope: + # The DeepseekV2 version of RoPE is different than upstream LLaMA. + # See https://github.com/linkedin/Liger-Kernel/issues/129#issuecomment-2313763528 logging.warning("Fused liger_rope is not supported for DeepseekV2.") if cfg.liger_rms_norm: modeling_mod.DeepseekV2RMSNorm = LigerRMSNorm