diff --git a/examples/language-modeling/run_clm.py b/examples/language-modeling/run_clm.py
index 396631b9ff9311..2abdecdd1b2e77 100644
--- a/examples/language-modeling/run_clm.py
+++ b/examples/language-modeling/run_clm.py
@@ -313,9 +313,12 @@ def group_texts(examples):
 
     # Training
     if training_args.do_train:
-        trainer.train(
-            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
+        model_path = (
+            model_args.model_name_or_path
+            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
+            else None
         )
+        trainer.train(model_path=model_path)
         trainer.save_model()  # Saves the tokenizer too for easy upload
 
     # Evaluation
diff --git a/examples/language-modeling/run_mlm.py b/examples/language-modeling/run_mlm.py
index dfc2614a72f526..664128eaf9fd94 100644
--- a/examples/language-modeling/run_mlm.py
+++ b/examples/language-modeling/run_mlm.py
@@ -354,9 +354,12 @@ def group_texts(examples):
 
     # Training
     if training_args.do_train:
-        trainer.train(
-            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
+        model_path = (
+            model_args.model_name_or_path
+            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
+            else None
         )
+        trainer.train(model_path=model_path)
         trainer.save_model()  # Saves the tokenizer too for easy upload
 
     # Evaluation
diff --git a/examples/language-modeling/run_mlm_wwm.py b/examples/language-modeling/run_mlm_wwm.py
index b2ffcc34ac942a..e7c6505fc93edb 100644
--- a/examples/language-modeling/run_mlm_wwm.py
+++ b/examples/language-modeling/run_mlm_wwm.py
@@ -302,9 +302,12 @@ def tokenize_function(examples):
 
     # Training
     if training_args.do_train:
-        trainer.train(
-            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
+        model_path = (
+            model_args.model_name_or_path
+            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
+            else None
         )
+        trainer.train(model_path=model_path)
         trainer.save_model()  # Saves the tokenizer too for easy upload
 
     # Evaluation
diff --git a/examples/language-modeling/run_plm.py b/examples/language-modeling/run_plm.py
index 65700a415c8bcf..0e264115d8f2c6 100644
--- a/examples/language-modeling/run_plm.py
+++ b/examples/language-modeling/run_plm.py
@@ -344,9 +344,12 @@ def group_texts(examples):
 
     # Training
     if training_args.do_train:
-        trainer.train(
-            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
+        model_path = (
+            model_args.model_name_or_path
+            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
+            else None
         )
+        trainer.train(model_path=model_path)
         trainer.save_model()  # Saves the tokenizer too for easy upload
 
     # Evaluation
diff --git a/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py b/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py
index a659ed95bba68e..cefa064cadbfd8 100644
--- a/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py
+++ b/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py
@@ -307,9 +307,18 @@ def tokenize_function(examples):
 
     # Training
     if training_args.do_train:
+{%- if cookiecutter.can_train_from_scratch == "False" %}
         trainer.train(
             model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
         )
+{%- elif cookiecutter.can_train_from_scratch == "True" %}
+        model_path = (
+            model_args.model_name_or_path
+            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
+            else None
+        )
+        trainer.train(model_path=model_path)
+{% endif %}
         trainer.save_model()  # Saves the tokenizer too for easy upload
 
     # Evaluation