diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py
index 05d74d654252..84983cb3ce25 100644
--- a/src/transformers/modeling_utils.py
+++ b/src/transformers/modeling_utils.py
@@ -513,6 +513,7 @@ def load_state_dict(checkpoint_file: Union[str, os.PathLike]):
             is_deepspeed_zero3_enabled() and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0
         ) or (is_fsdp_enabled() and not is_local_dist_rank_0()):
             map_location = "meta"
+            assert(0)
         else:
             map_location = "cpu"