From e0fe274d5ea5a69df7ce50ebe545d13f33618c72 Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Wed, 20 Nov 2024 14:46:55 -0500 Subject: [PATCH] mean_resizing = True does not work with mixed/meta initialization --- composer/models/huggingface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer/models/huggingface.py b/composer/models/huggingface.py index 72550c12e2..59a82363c6 100644 --- a/composer/models/huggingface.py +++ b/composer/models/huggingface.py @@ -167,7 +167,7 @@ def _check_tokenizer_and_maybe_resize_embeddings(self, allow_embedding_resizing: f' This would cause an error during training.' f' Resizing the model embeddings to {len(self.tokenizer)} from {self.config.vocab_size}.', ) - self.model.resize_token_embeddings(len(self.tokenizer)) + self.model.resize_token_embeddings(len(self.tokenizer), mean_resizing=False) else: raise ValueError( f'The number of tokens in the tokenizer is greater than the number of tokens in the model.'