diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py
index f54e1bbdb69eed..f66b5734bd9808 100755
--- a/src/transformers/configuration_utils.py
+++ b/src/transformers/configuration_utils.py
@@ -676,6 +676,10 @@ def from_dict(cls, config_dict: Dict[str, Any], **kwargs) -> "PretrainedConfig":
             [`PretrainedConfig`]: The configuration object instantiated from those parameters.
         """
         return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)
+        # Those arguments may be passed along for our internal telemetry.
+        # We remove them so they don't appear in `return_unused_kwargs`.
+        kwargs.pop("_from_auto", None)
+        kwargs.pop("_from_pipeline", None)
 
         config = cls(**config_dict)
 
diff --git a/tests/test_configuration_common.py b/tests/test_configuration_common.py
index d17ff540679230..853a19c3ec8419 100644
--- a/tests/test_configuration_common.py
+++ b/tests/test_configuration_common.py
@@ -356,7 +356,7 @@ def test_repo_versioning_before(self):
         )
         self.assertEqual(new_configuration.hidden_size, 2)
         # This checks `_configuration_file` ia not kept in the kwargs by mistake.
-        self.assertDictEqual(kwargs, {"_from_auto": True})
+        self.assertDictEqual(kwargs, {})
 
         # Testing an older version by monkey-patching the version in the module it's used.
         import transformers as old_transformers