diff --git a/src/transformers/utils/import_utils.py b/src/transformers/utils/import_utils.py
index 895c54fd542511..9e4b1c3fc90c36 100644
--- a/src/transformers/utils/import_utils.py
+++ b/src/transformers/utils/import_utils.py
@@ -42,16 +42,20 @@ def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[
     # Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version
     package_exists = importlib.util.find_spec(pkg_name) is not None
     package_version = "N/A"
-    if package_exists:
-        try:
-            package_version = importlib.metadata.version(pkg_name)
-            package_exists = True
-        except importlib.metadata.PackageNotFoundError:
-            package_exists = False
-        logger.debug(f"Detected {pkg_name} version {package_version}")
     if return_version:
+        if package_exists:
+            try:
+                package_version = importlib.metadata.version(pkg_name)
+                package_exists = True
+            except importlib.metadata.PackageNotFoundError:
+                package_exists = False
+        logger.debug(f"Detected {pkg_name} version {package_version}")
         return package_exists, package_version
     else:
+        if package_exists:
+            package_exists = True
+        else:
+            package_exists = False
         return package_exists