Make normalization capitalization insensitive.

NVIDIA · Dec 20, 2023 · 4a4e725 · 4a4e725
1 parent c1b0eb0
commit 4a4e725
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/nemo/collections/nlp/models/language_modeling/megatron_gpt_model.py b/nemo/collections/nlp/models/language_modeling/megatron_gpt_model.py
@@ -1518,7 +1518,7 @@ def build_transformer_config(self) -> TransformerConfig:
         gated_linear_unit = activation.endswith('glu')
         activation_func = activation_to_func(activation)
 
-        normalization = self.cfg.get('normalization', 'layernorm')
+        normalization = self.cfg.get('normalization', 'layernorm').lower()
         layernorm_zero_centered_gamma = self.cfg.get('normalization', 'layernorm') == 'layernorm1p'
         if normalization == 'layernorm':
             normalization = 'LayerNorm'