From 9edbb38a6d2fd45131f9eb2e406dcdcbc45b41df Mon Sep 17 00:00:00 2001 From: gouzi <530971494@qq.com> Date: Thu, 28 Mar 2024 22:26:36 +0800 Subject: [PATCH 1/4] support liblapack arm --- cmake/external/lapack.cmake | 41 +++++++++++++------ paddle/phi/backends/dynload/dynamic_loader.cc | 26 +++++++++++- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/cmake/external/lapack.cmake b/cmake/external/lapack.cmake index 62da0987085d12..75bed8574369a2 100644 --- a/cmake/external/lapack.cmake +++ b/cmake/external/lapack.cmake @@ -48,19 +48,34 @@ elseif(WIN32) set(GFORTRAN_LIB "${LAPACK_LIB_DIR}/libgfortran-3.dll") set(BLAS_LIB "${LAPACK_LIB_DIR}/libblas.dll") set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.dll") -else() - set(LAPACK_FILE - "lapack_mac_v3.10.0.20210628.tar.gz" - CACHE STRING "" FORCE) - set(LAPACK_URL - "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}" - CACHE STRING "" FORCE) - set(LAPACK_URL_MD5 427aecf8dee8523de3566ca8e47944d7) - set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.0.dylib") - set(GNU_RT_LIB_2 "${LAPACK_LIB_DIR}/libgcc_s.1.dylib") - set(GFORTRAN_LIB "${LAPACK_LIB_DIR}/libgfortran.5.dylib") - set(BLAS_LIB "${LAPACK_LIB_DIR}/libblas.3.dylib") - set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.3.dylib") +else() # MacOS + if(APPLE AND WITH_ARM) + set(LAPACK_FILE + "lapack_mac_arm64_v0.3.26.tar.gz" + CACHE STRING "" FORCE) + set(LAPACK_URL + "https://github.com/PaddlePaddle/Paddle/files/14790239/${LAPACK_FILE}" + CACHE STRING "" FORCE) + set(LAPACK_URL_MD5 3f6412105ae2b7465e5ee90c8673e6d4) + set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.0.dylib") + set(GNU_RT_LIB_2 "${LAPACK_LIB_DIR}/libgcc_s.1.dylib") + set(GFORTRAN_LIB "${LAPACK_LIB_DIR}/libgfortran.5.dylib") + set(BLAS_LIB "${LAPACK_LIB_DIR}/libblas.dylib") + set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.dylib") + else() + set(LAPACK_FILE + "lapack_mac_v3.10.0.20210628.tar.gz" + CACHE STRING "" FORCE) + set(LAPACK_URL + "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}" + CACHE STRING "" FORCE) + set(LAPACK_URL_MD5 427aecf8dee8523de3566ca8e47944d7) + set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.0.dylib") + set(GNU_RT_LIB_2 "${LAPACK_LIB_DIR}/libgcc_s.1.dylib") + set(GFORTRAN_LIB "${LAPACK_LIB_DIR}/libgfortran.5.dylib") + set(BLAS_LIB "${LAPACK_LIB_DIR}/libblas.3.dylib") + set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.3.dylib") + endif() endif() function(download_lapack) diff --git a/paddle/phi/backends/dynload/dynamic_loader.cc b/paddle/phi/backends/dynload/dynamic_loader.cc index 7f8e00b4d9e6cd..e1036612a79d7f 100644 --- a/paddle/phi/backends/dynload/dynamic_loader.cc +++ b/paddle/phi/backends/dynload/dynamic_loader.cc @@ -182,6 +182,17 @@ static inline void* GetDsoHandleFromSpecificPath(const std::string& spec_path, return dso_handle; } +static inline std::string FindFiles(const std::filesystem::path& directory, + const std::string& filename) { + for (const auto& entry : + std::filesystem::recursive_directory_iterator(directory)) { + if (entry.is_regular_file() && entry.path().filename() == filename) { + return entry.path(); + } + } + return ""; +} + static inline void* GetDsoHandleFromDefaultPath(const std::string& dso_path, int dynload_flags) { // default search from LD_LIBRARY_PATH/DYLD_LIBRARY_PATH @@ -195,10 +206,17 @@ static inline void* GetDsoHandleFromDefaultPath(const std::string& dso_path, // bring System Integrity Projection (SIP), if dso_handle // is null, search from default package path in Mac OS. #if defined(__APPLE__) || defined(__OSX__) +#if defined(__arm__) || defined(__aarch64__) + if (nullptr == dso_handle) { + dso_handle = dlopen(FindFiles("/opt/homebrew/Cellar/", dso_path).c_str(), + dynload_flags); + } +#else if (nullptr == dso_handle) { - dso_handle = - dlopen(join("/usr/local/cuda/lib/", dso_path).c_str(), dynload_flags); + dso_handle = dlopen(FindFiles("/usr/local/cuda/lib/", dso_path).c_str(), + dynload_flags); } +#endif #endif return dso_handle; @@ -618,7 +636,11 @@ void* GetMKLMLDsoHandle() { void* GetLAPACKDsoHandle() { #if defined(__APPLE__) || defined(__OSX__) +#if defined(__arm__) || defined(__aarch64__) + return GetDsoHandleFromSearchPath(FLAGS_lapack_dir, "liblapack.dylib"); +#else return GetDsoHandleFromSearchPath(FLAGS_lapack_dir, "liblapack.3.dylib"); +#endif #elif defined(_WIN32) return GetDsoHandleFromSearchPath(FLAGS_lapack_dir, "liblapack.dll"); #else From cf4b0e9444f329eaad19e7cca1a9d64fd289ec6f Mon Sep 17 00:00:00 2001 From: gouzi <530971494@qq.com> Date: Thu, 28 Mar 2024 22:33:58 +0800 Subject: [PATCH 2/4] update url --- cmake/external/lapack.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/lapack.cmake b/cmake/external/lapack.cmake index 75bed8574369a2..96472583c63f2d 100644 --- a/cmake/external/lapack.cmake +++ b/cmake/external/lapack.cmake @@ -54,7 +54,7 @@ else() # MacOS "lapack_mac_arm64_v0.3.26.tar.gz" CACHE STRING "" FORCE) set(LAPACK_URL - "https://github.com/PaddlePaddle/Paddle/files/14790239/${LAPACK_FILE}" + "https://github.com/PaddlePaddle/Paddle/files/14791051/${LAPACK_FILE}" CACHE STRING "" FORCE) set(LAPACK_URL_MD5 3f6412105ae2b7465e5ee90c8673e6d4) set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.0.dylib") From 6ee2afcdfcc07092837ed22a688b42770641ed7d Mon Sep 17 00:00:00 2001 From: gouzi <530971494@qq.com> Date: Fri, 29 Mar 2024 01:03:30 +0800 Subject: [PATCH 3/4] fix FindLibAbsolutePath --- paddle/phi/backends/dynload/dynamic_loader.cc | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/paddle/phi/backends/dynload/dynamic_loader.cc b/paddle/phi/backends/dynload/dynamic_loader.cc index e1036612a79d7f..0b056d6df972fb 100644 --- a/paddle/phi/backends/dynload/dynamic_loader.cc +++ b/paddle/phi/backends/dynload/dynamic_loader.cc @@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/phi/backends/dynload/dynamic_loader.h" +#include #include #include @@ -182,13 +183,30 @@ static inline void* GetDsoHandleFromSpecificPath(const std::string& spec_path, return dso_handle; } -static inline std::string FindFiles(const std::filesystem::path& directory, - const std::string& filename) { - for (const auto& entry : - std::filesystem::recursive_directory_iterator(directory)) { - if (entry.is_regular_file() && entry.path().filename() == filename) { - return entry.path(); +static inline std::string FindLibAbsolutePath(const std::string& directory, + const std::string& filename) { + DIR* dir; + struct dirent* ent; + + if ((dir = opendir(directory.c_str())) != nullptr) { + while ((ent = readdir(dir)) != nullptr) { + if (ent->d_type == DT_REG || ent->d_type == DT_LNK) { + if (filename == std::string(ent->d_name)) { + closedir(dir); + return join(directory, ent->d_name); + } + } else if (ent->d_type == DT_DIR) { + if (strcmp(ent->d_name, ".") != 0 && strcmp(ent->d_name, "..") != 0) { + std::string res = + FindLibAbsolutePath(join(directory, ent->d_name) + "/", filename); + if (!res.empty()) { + closedir(dir); + return res; + } + } + } } + closedir(dir); } return ""; } @@ -208,13 +226,15 @@ static inline void* GetDsoHandleFromDefaultPath(const std::string& dso_path, #if defined(__APPLE__) || defined(__OSX__) #if defined(__arm__) || defined(__aarch64__) if (nullptr == dso_handle) { - dso_handle = dlopen(FindFiles("/opt/homebrew/Cellar/", dso_path).c_str(), - dynload_flags); + dso_handle = + dlopen(FindLibAbsolutePath("/opt/homebrew/Cellar/", dso_path).c_str(), + dynload_flags); } #else if (nullptr == dso_handle) { - dso_handle = dlopen(FindFiles("/usr/local/cuda/lib/", dso_path).c_str(), - dynload_flags); + dso_handle = + dlopen(FindLibAbsolutePath("/usr/local/cuda/lib/", dso_path).c_str(), + dynload_flags); } #endif #endif From 1f011b50c55cad7bce41fb0492a946da1d87b701 Mon Sep 17 00:00:00 2001 From: gouzi <530971494@qq.com> Date: Fri, 29 Mar 2024 10:42:17 +0800 Subject: [PATCH 4/4] update url -> bcebos --- cmake/external/lapack.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/lapack.cmake b/cmake/external/lapack.cmake index 96472583c63f2d..2865dabdaccce2 100644 --- a/cmake/external/lapack.cmake +++ b/cmake/external/lapack.cmake @@ -54,7 +54,7 @@ else() # MacOS "lapack_mac_arm64_v0.3.26.tar.gz" CACHE STRING "" FORCE) set(LAPACK_URL - "https://github.com/PaddlePaddle/Paddle/files/14791051/${LAPACK_FILE}" + "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}" CACHE STRING "" FORCE) set(LAPACK_URL_MD5 3f6412105ae2b7465e5ee90c8673e6d4) set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.0.dylib")