diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index 78acda4f15c..8a40d650115 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -2245,7 +2245,9 @@ def evaluation_loop(
         # Number of samples
         if not isinstance(eval_dataset, IterableDataset):
             num_samples = len(eval_dataset)
-        elif isinstance(eval_dataset, IterableDatasetShard):
+        # The instance check is weird and does not actually check for the type, but whether the dataset has the right
+        # methods. Therefore we need to make sure it also has the attribute.
+        elif isinstance(eval_dataset, IterableDatasetShard) and hasattr(eval_dataset, "num_examples"):
             num_samples = eval_dataset.num_examples
         else:
             num_samples = observed_num_examples