diff --git a/python/setup.py b/python/setup.py
index ad0ef6ffde49..7f76a7b9f2af 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -129,6 +129,12 @@ def _remove_path(path):
 LIB_LIST, __version__ = get_lib_path()
 __version__ = git_describe_version(__version__)
 
+if not CONDA_BUILD and not INPLACE_BUILD:
+    # Wheel cleanup
+    for path in LIB_LIST:
+        libname = os.path.basename(path)
+        _remove_path(f"tvm/{libname}")
+
 
 def config_cython():
     """Try to configure cython and return cython configuration"""
@@ -260,5 +266,5 @@ def long_description_contents():
     # Wheel cleanup
     os.remove("MANIFEST.in")
     for path in LIB_LIST:
-        _, libname = os.path.split(path)
+        libname = os.path.basename(path)
         _remove_path(f"tvm/{libname}")
diff --git a/python/tvm/_ffi/libinfo.py b/python/tvm/_ffi/libinfo.py
index 3bbc588f451e..77dad46c82e4 100644
--- a/python/tvm/_ffi/libinfo.py
+++ b/python/tvm/_ffi/libinfo.py
@@ -143,8 +143,18 @@ def find_lib_path(name=None, search_path=None, optional=False):
             ]
 
         name = lib_dll_names + runtime_dll_names + ext_lib_dll_names
-        lib_dll_path = [os.path.join(p, name) for name in lib_dll_names for p in dll_path]
-        runtime_dll_path = [os.path.join(p, name) for name in runtime_dll_names for p in dll_path]
+        lib_dll_path = [
+            os.path.join(p, name)
+            for name in lib_dll_names
+            for p in dll_path
+            if not p.endswith("python/tvm")
+        ]
+        runtime_dll_path = [
+            os.path.join(p, name)
+            for name in runtime_dll_names
+            for p in dll_path
+            if not p.endswith("python/tvm")
+        ]
         ext_lib_dll_path = [os.path.join(p, name) for name in ext_lib_dll_names for p in dll_path]
     if not use_runtime:
         # try to find lib_dll_path