[Bugfix] Correct LayerNorm epsilon parameter in modernbert.py (vllm-project#27008)

bogdanminko · gemini-code-assist[bot] · xuebwang-amd · commit d5817c0e76cf · 2025-10-24T09:27:26.000Z
Signed-off-by: bogdanm &lt;152898065+bogdan01m@users.noreply.github.com&gt;
Co-authored-by: gemini-code-assist[bot] &lt;176961590+gemini-code-assist[bot]@users.noreply.github.com&gt;
Signed-off-by: xuebwang-amd &lt;xuebwang@amd.com&gt;
diff --git a/vllm/model_executor/models/modernbert.py b/vllm/model_executor/models/modernbert.py
@@ -39,9 +39,12 @@ def __init__(self, config: ModernBertConfig):
         self.tok_embeddings = VocabParallelEmbedding(
             config.vocab_size, config.hidden_size
         )
-        self.norm = nn.LayerNorm(
-            config.hidden_size, eps=config.layer_norm_eps, bias=config.norm_bias
+        eps = (
+            getattr(config, "norm_eps", None)
+            or getattr(config, "layer_norm_eps", None)
+            or 1e-5
         )
+        self.norm = nn.LayerNorm(config.hidden_size, eps=eps, bias=config.norm_bias)
 
     def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
         return self.tok_embeddings(input_ids)

Original file line number	Diff line number	Diff line change
`@@ -39,9 +39,12 @@ def __init__(self, config: ModernBertConfig):`
`39`	`39`	`self.tok_embeddings = VocabParallelEmbedding(`
`40`	`40`	`config.vocab_size, config.hidden_size`
`41`	`41`	`)`
`42`		`- self.norm = nn.LayerNorm(`
`43`		`- config.hidden_size, eps=config.layer_norm_eps, bias=config.norm_bias`
	`42`	`+ eps = (`
	`43`	`+ getattr(config, "norm_eps", None)`
	`44`	`+ or getattr(config, "layer_norm_eps", None)`
	`45`	`+ or 1e-5`
`44`	`46`	`)`
	`47`	`+ self.norm = nn.LayerNorm(config.hidden_size, eps=eps, bias=config.norm_bias)`
`45`	`48`
`46`	`49`	`def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:`
`47`	`50`	`return self.tok_embeddings(input_ids)`