diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index 2ac88349484c80..bbe10a5ee30e9d 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -729,6 +729,7 @@ def train(self, model_path: Optional[str] = None, trial: Union["optuna.Trial", D
 
         tr_loss = torch.tensor(0.0).to(self.args.device)
         self._logging_loss_scalar = 0
+        self._globalstep_last_logged = 0
         self._total_flos = self.state.total_flos
         model.zero_grad()
 
@@ -849,7 +850,9 @@ def _maybe_log_save_evaluate(self, tr_loss, model, trial, epoch):
         if self.control.should_log:
             logs: Dict[str, float] = {}
             tr_loss_scalar = tr_loss.item()
-            logs["loss"] = (tr_loss_scalar - self._logging_loss_scalar) / self.args.logging_steps
+            logs["loss"] = (tr_loss_scalar - self._logging_loss_scalar) / (
+                self.state.global_step - self._globalstep_last_logged
+            )
             # backward compatibility for pytorch schedulers
             logs["learning_rate"] = (
                 self.lr_scheduler.get_last_lr()[0]
@@ -857,6 +860,7 @@ def _maybe_log_save_evaluate(self, tr_loss, model, trial, epoch):
                 else self.lr_scheduler.get_lr()[0]
             )
             self._logging_loss_scalar = tr_loss_scalar
+            self._globalstep_last_logged = self.state.global_step
 
             self.log(logs)
 
diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py
index 2fc6ffd710387a..2442519084a611 100644
--- a/src/transformers/training_args.py
+++ b/src/transformers/training_args.py
@@ -250,7 +250,7 @@ class TrainingArguments:
     warmup_steps: int = field(default=0, metadata={"help": "Linear warmup over warmup_steps."})
 
     logging_dir: Optional[str] = field(default_factory=default_logdir, metadata={"help": "Tensorboard log dir."})
-    logging_first_step: bool = field(default=False, metadata={"help": "Log and eval the first global_step"})
+    logging_first_step: bool = field(default=False, metadata={"help": "Log the first global_step"})
     logging_steps: int = field(default=500, metadata={"help": "Log every X updates steps."})
     save_steps: int = field(default=500, metadata={"help": "Save checkpoint every X updates steps."})
     save_total_limit: Optional[int] = field(