diff --git a/python/paddle/base/core.py b/python/paddle/base/core.py index f0bd0b089c2839..b25812fa2c769f 100644 --- a/python/paddle/base/core.py +++ b/python/paddle/base/core.py @@ -403,11 +403,7 @@ def set_paddle_custom_device_lib_path(lib_path): # set paddle lib path def set_paddle_lib_path(): - site_dirs = ( - site.getsitepackages() - if hasattr(site, 'getsitepackages') - else [x for x in sys.path if 'site-packages' in x] - ) + site_dirs = site.getsitepackages() for site_dir in site_dirs: lib_dir = os.path.sep.join([site_dir, 'paddle', 'libs']) if os.path.exists(lib_dir): diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py index 1a13fad34b1db3..72f9e930585f2d 100644 --- a/python/paddle/utils/cpp_extension/extension_utils.py +++ b/python/paddle/utils/cpp_extension/extension_utils.py @@ -821,23 +821,33 @@ def find_rocm_includes(): return [os.path.join(rocm_home, 'include')] -def find_paddle_includes(use_cuda=False): +def _get_all_paddle_includes_from_include_root(include_root: str) -> list[str]: """ - Return Paddle necessary include dir path. + Get all paddle include directories from include root (packaged in wheel) """ - # pythonXX/site-packages/paddle/include - paddle_include_dir = get_include() - third_party_dir = os.path.join(paddle_include_dir, 'third_party') - include_dirs = [paddle_include_dir, third_party_dir] + third_party_dir = os.path.join(include_root, 'third_party') + include_dirs = [include_root, third_party_dir] if not IS_WINDOWS: compat_dir_root = os.path.join( - paddle_include_dir, 'paddle/phi/api/include/compat' + include_root, 'paddle/phi/api/include/compat' ) compat_dir_api_include = os.path.join( - paddle_include_dir, + include_root, 'paddle/phi/api/include/compat/torch/csrc/api/include', ) include_dirs.extend([compat_dir_root, compat_dir_api_include]) + return include_dirs + + +def find_paddle_includes(use_cuda=False): + """ + Return Paddle necessary include dir path. + """ + # pythonXX/site-packages/paddle/include + paddle_include_dir = get_include() + include_dirs = _get_all_paddle_includes_from_include_root( + paddle_include_dir + ) if use_cuda: if core.is_compiled_with_rocm(): diff --git a/test/auto_parallel/custom_op/utils.py b/test/auto_parallel/custom_op/utils.py index 05047c168fc29b..999c368d509a8b 100644 --- a/test/auto_parallel/custom_op/utils.py +++ b/test/auto_parallel/custom_op/utils.py @@ -16,7 +16,9 @@ from pathlib import Path from site import getsitepackages -from paddle.utils.cpp_extension.extension_utils import IS_WINDOWS +from paddle.utils.cpp_extension.extension_utils import ( + _get_all_paddle_includes_from_include_root, +) # Test for extra compile args extra_cc_args = ['-w', '-g'] @@ -38,18 +40,9 @@ def get_paddle_includes(): for site_packages_path in getsitepackages(): paddle_include_dir = Path(site_packages_path) / "paddle/include" - paddle_includes.append(str(paddle_include_dir)) - paddle_includes.append(str(paddle_include_dir / 'third_party')) - if not IS_WINDOWS: - paddle_includes.append( - str(paddle_include_dir / 'paddle/phi/api/include/compat') - ) - paddle_includes.append( - str( - paddle_include_dir - / 'paddle/phi/api/include/compat/torch/csrc/api/include' - ) - ) + paddle_includes.extend( + _get_all_paddle_includes_from_include_root(str(paddle_include_dir)) + ) return paddle_includes diff --git a/test/auto_parallel/semi_auto_parallel_for_custom_relu.py b/test/auto_parallel/semi_auto_parallel_for_custom_relu.py index 78845789f713ee..85daa3c0876fa1 100644 --- a/test/auto_parallel/semi_auto_parallel_for_custom_relu.py +++ b/test/auto_parallel/semi_auto_parallel_for_custom_relu.py @@ -21,7 +21,11 @@ import paddle import paddle.distributed as dist from paddle.utils.cpp_extension import get_build_directory, load -from paddle.utils.cpp_extension.extension_utils import IS_WINDOWS, run_cmd +from paddle.utils.cpp_extension.extension_utils import ( + IS_WINDOWS, + _get_all_paddle_includes_from_include_root, + run_cmd, +) # Note(Aurelius84): We use `add_test` in Cmake to config how to run unittest in CI. # `PYTHONPATH` will be set as `build/python/paddle` that will make no way to find @@ -30,18 +34,10 @@ paddle_includes = [] for site_packages_path in getsitepackages(): paddle_include_dir = Path(site_packages_path) / "paddle/include" - paddle_includes.append(str(paddle_include_dir)) - paddle_includes.append(str(paddle_include_dir / 'third_party')) - if not IS_WINDOWS: - paddle_includes.append( - str(paddle_include_dir / 'paddle/phi/api/include/compat') - ) - paddle_includes.append( - str( - paddle_include_dir - / 'paddle/phi/api/include/compat/torch/csrc/api/include' - ) - ) + paddle_includes.extend( + _get_all_paddle_includes_from_include_root(str(paddle_include_dir)) + ) + # Test for extra compile args extra_cc_args = ['-w', '-g'] if not IS_WINDOWS else ['/w'] diff --git a/test/auto_parallel/semi_auto_parallel_simple_net_custom_relu.py b/test/auto_parallel/semi_auto_parallel_simple_net_custom_relu.py index 54b2452bced96c..07523769297491 100644 --- a/test/auto_parallel/semi_auto_parallel_simple_net_custom_relu.py +++ b/test/auto_parallel/semi_auto_parallel_simple_net_custom_relu.py @@ -23,7 +23,11 @@ import paddle.nn.functional as F from paddle import nn from paddle.utils.cpp_extension import get_build_directory, load -from paddle.utils.cpp_extension.extension_utils import IS_WINDOWS, run_cmd +from paddle.utils.cpp_extension.extension_utils import ( + IS_WINDOWS, + _get_all_paddle_includes_from_include_root, + run_cmd, +) # Note(Aurelius84): We use `add_test` in Cmake to config how to run unittest in CI. # `PYTHONPATH` will be set as `build/python/paddle` that will make no way to find @@ -32,18 +36,10 @@ paddle_includes = [] for site_packages_path in getsitepackages(): paddle_include_dir = Path(site_packages_path) / "paddle/include" - paddle_includes.append(str(paddle_include_dir)) - paddle_includes.append(str(paddle_include_dir / 'third_party')) - if not IS_WINDOWS: - paddle_includes.append( - str(paddle_include_dir / 'paddle/phi/api/include/compat') - ) - paddle_includes.append( - str( - paddle_include_dir - / 'paddle/phi/api/include/compat/torch/csrc/api/include' - ) - ) + paddle_includes.extend( + _get_all_paddle_includes_from_include_root(str(paddle_include_dir)) + ) + # Test for extra compile args extra_cc_args = ['-w', '-g'] if not IS_WINDOWS else ['/w'] diff --git a/test/cpp_extension/cpp_extension_setup.py b/test/cpp_extension/cpp_extension_setup.py index f9d168f7a346a4..c1af6112545a2f 100644 --- a/test/cpp_extension/cpp_extension_setup.py +++ b/test/cpp_extension/cpp_extension_setup.py @@ -20,23 +20,17 @@ import paddle from paddle.utils.cpp_extension import CppExtension, CUDAExtension, setup -from paddle.utils.cpp_extension.extension_utils import IS_WINDOWS +from paddle.utils.cpp_extension.extension_utils import ( + _get_all_paddle_includes_from_include_root, +) paddle_includes = [] for site_packages_path in getsitepackages(): paddle_include_dir = Path(site_packages_path) / "paddle/include" - paddle_includes.append(str(paddle_include_dir)) - paddle_includes.append(str(paddle_include_dir / 'third_party')) - if not IS_WINDOWS: - paddle_includes.append( - str(paddle_include_dir / 'paddle/phi/api/include/compat') - ) - paddle_includes.append( - str( - paddle_include_dir - / 'paddle/phi/api/include/compat/torch/csrc/api/include' - ) - ) + paddle_includes.extend( + _get_all_paddle_includes_from_include_root(str(paddle_include_dir)) + ) + # Add current dir, search custom_power.h paddle_includes.append(os.path.dirname(os.path.abspath(__file__))) diff --git a/test/cpp_extension/test_cpp_extension_jit.py b/test/cpp_extension/test_cpp_extension_jit.py index dfedce266354a9..56c82f2607be41 100644 --- a/test/cpp_extension/test_cpp_extension_jit.py +++ b/test/cpp_extension/test_cpp_extension_jit.py @@ -23,7 +23,9 @@ import paddle from paddle.utils.cpp_extension import load -from paddle.utils.cpp_extension.extension_utils import IS_WINDOWS +from paddle.utils.cpp_extension.extension_utils import ( + _get_all_paddle_includes_from_include_root, +) if os.name == 'nt' or sys.platform.startswith('darwin'): # only support Linux now @@ -37,18 +39,10 @@ paddle_includes = [] for site_packages_path in getsitepackages(): paddle_include_dir = Path(site_packages_path) / "paddle/include" - paddle_includes.append(str(paddle_include_dir)) - paddle_includes.append(str(paddle_include_dir / 'third_party')) - if not IS_WINDOWS: - paddle_includes.append( - str(paddle_include_dir / 'paddle/phi/api/include/compat') - ) - paddle_includes.append( - str( - paddle_include_dir - / 'paddle/phi/api/include/compat/torch/csrc/api/include' - ) - ) + paddle_includes.extend( + _get_all_paddle_includes_from_include_root(str(paddle_include_dir)) + ) + # include "custom_power.h" paddle_includes.append(os.path.dirname(os.path.abspath(__file__))) diff --git a/test/cpp_extension/utils.py b/test/cpp_extension/utils.py index eb1aab0d0f5205..79ebb8e2d70a5c 100644 --- a/test/cpp_extension/utils.py +++ b/test/cpp_extension/utils.py @@ -18,7 +18,10 @@ import numpy as np -from paddle.utils.cpp_extension.extension_utils import IS_WINDOWS +from paddle.utils.cpp_extension.extension_utils import ( + IS_WINDOWS, + _get_all_paddle_includes_from_include_root, +) IS_MAC = sys.platform.startswith('darwin') @@ -29,18 +32,10 @@ paddle_includes = [] for site_packages_path in getsitepackages(): paddle_include_dir = Path(site_packages_path) / "paddle/include" - paddle_includes.append(str(paddle_include_dir)) - paddle_includes.append(str(paddle_include_dir / 'third_party')) - if not IS_WINDOWS: - paddle_includes.append( - str(paddle_include_dir / 'paddle/phi/api/include/compat') - ) - paddle_includes.append( - str( - paddle_include_dir - / 'paddle/phi/api/include/compat/torch/csrc/api/include' - ) - ) + paddle_includes.extend( + _get_all_paddle_includes_from_include_root(str(paddle_include_dir)) + ) + # Test for extra compile args extra_cc_args = ['-w', '-g'] if not IS_WINDOWS else ['/w'] diff --git a/test/custom_kernel/test_custom_kernel_load.py b/test/custom_kernel/test_custom_kernel_load.py index 0c7952d3648ad6..dcf0bdc8eca8bc 100644 --- a/test/custom_kernel/test_custom_kernel_load.py +++ b/test/custom_kernel/test_custom_kernel_load.py @@ -31,11 +31,7 @@ def setUp(self): # get paddle lib path and place so paddle_lib_path = '' - site_dirs = ( - site.getsitepackages() - if hasattr(site, 'getsitepackages') - else [x for x in sys.path if 'site-packages' in x] - ) + site_dirs = site.getsitepackages() for site_dir in site_dirs: lib_dir = os.path.sep.join([site_dir, 'paddle', 'libs']) if os.path.exists(lib_dir): diff --git a/test/custom_op/utils.py b/test/custom_op/utils.py index 06f81768d10c98..831a460f908310 100644 --- a/test/custom_op/utils.py +++ b/test/custom_op/utils.py @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import sys from pathlib import Path from site import getsitepackages import numpy as np -from paddle.utils.cpp_extension.extension_utils import IS_WINDOWS +from paddle.utils.cpp_extension.extension_utils import ( + IS_WINDOWS, + _get_all_paddle_includes_from_include_root, +) IS_MAC = sys.platform.startswith('darwin') @@ -31,19 +33,11 @@ paddle_libraries = [] for site_packages_path in getsitepackages(): paddle_include_dir = Path(site_packages_path) / "paddle/include" - paddle_includes.append(str(paddle_include_dir)) - paddle_includes.append(str(paddle_include_dir / 'third_party')) - if not IS_WINDOWS: - paddle_includes.append( - str(paddle_include_dir / 'paddle/phi/api/include/compat') - ) - paddle_includes.append( - str( - paddle_include_dir - / 'paddle/phi/api/include/compat/torch/csrc/api/include' - ) - ) - paddle_libraries.append(os.path.join(site_packages_path, 'paddle', 'libs')) + paddle_includes.extend( + _get_all_paddle_includes_from_include_root(str(paddle_include_dir)) + ) + + paddle_libraries.append(str(Path(site_packages_path) / 'paddle' / 'libs')) # Test for extra compile args extra_cc_args = ['-w', '-g'] if not IS_WINDOWS else ['/w'] diff --git a/test/custom_runtime/test_custom_op_setup.py b/test/custom_runtime/test_custom_op_setup.py index 51834e114654f7..a48cef5a53081b 100644 --- a/test/custom_runtime/test_custom_op_setup.py +++ b/test/custom_runtime/test_custom_op_setup.py @@ -21,7 +21,9 @@ import numpy as np -from paddle.utils.cpp_extension.extension_utils import IS_WINDOWS +from paddle.utils.cpp_extension.extension_utils import ( + _get_all_paddle_includes_from_include_root, +) def custom_relu_dynamic(func, device, dtype, np_x, use_func=True): @@ -140,18 +142,11 @@ def setUp(self): paddle_includes = [] for site_packages_path in getsitepackages(): paddle_include_dir = Path(site_packages_path) / "paddle/include" - paddle_includes.append(str(paddle_include_dir)) - paddle_includes.append(str(paddle_include_dir / 'third_party')) - if not IS_WINDOWS: - paddle_includes.append( - str(paddle_include_dir / 'paddle/phi/api/include/compat') - ) - paddle_includes.append( - str( - paddle_include_dir - / 'paddle/phi/api/include/compat/torch/csrc/api/include' - ) + paddle_includes.extend( + _get_all_paddle_includes_from_include_root( + str(paddle_include_dir) ) + ) custom_module = paddle.utils.cpp_extension.load( name='custom_device', diff --git a/test/deprecated/custom_op/utils.py b/test/deprecated/custom_op/utils.py index 06f81768d10c98..831a460f908310 100644 --- a/test/deprecated/custom_op/utils.py +++ b/test/deprecated/custom_op/utils.py @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import sys from pathlib import Path from site import getsitepackages import numpy as np -from paddle.utils.cpp_extension.extension_utils import IS_WINDOWS +from paddle.utils.cpp_extension.extension_utils import ( + IS_WINDOWS, + _get_all_paddle_includes_from_include_root, +) IS_MAC = sys.platform.startswith('darwin') @@ -31,19 +33,11 @@ paddle_libraries = [] for site_packages_path in getsitepackages(): paddle_include_dir = Path(site_packages_path) / "paddle/include" - paddle_includes.append(str(paddle_include_dir)) - paddle_includes.append(str(paddle_include_dir / 'third_party')) - if not IS_WINDOWS: - paddle_includes.append( - str(paddle_include_dir / 'paddle/phi/api/include/compat') - ) - paddle_includes.append( - str( - paddle_include_dir - / 'paddle/phi/api/include/compat/torch/csrc/api/include' - ) - ) - paddle_libraries.append(os.path.join(site_packages_path, 'paddle', 'libs')) + paddle_includes.extend( + _get_all_paddle_includes_from_include_root(str(paddle_include_dir)) + ) + + paddle_libraries.append(str(Path(site_packages_path) / 'paddle' / 'libs')) # Test for extra compile args extra_cc_args = ['-w', '-g'] if not IS_WINDOWS else ['/w']