Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[cmake] add lapack,libmct,mklml third_party cache #54326

Merged
merged 8 commits into from
Jun 6, 2023
Merged

[cmake] add lapack,libmct,mklml third_party cache #54326

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ee06ead
[cmake] change lapack
gouzil Jun 3, 2023
3a2187d
[cmake] add file md5 ckeck
gouzil Jun 3, 2023
cc81267
[cmake] add libmct cache
gouzil Jun 3, 2023
c5cf793
[cmake] fix libmct no cmakelist.txt; rm lapack INSTALL_COMMAND
gouzil Jun 3, 2023
1302278
[cmake] add mklml third_party cache
gouzil Jun 3, 2023
b634123
[cmake] fix install error
gouzil Jun 3, 2023
63a8732
[cmake] fix install error
gouzil Jun 3, 2023
130b414
[cmake] Modify variable name
gouzil Jun 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 46 additions & 10 deletions cmake/external/lapack.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,19 @@
include(ExternalProject)

set(LAPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/lapack)
set(LAPACK_SOURCE_DIR ${THIRD_PARTY_PATH}/lapack/src/extern_lapack)
set(LAPACK_DOWNLOAD_DIR
${PADDLE_SOURCE_DIR}/third_party/lapack/${CMAKE_SYSTEM_NAME})
Comment on lines +18 to +19
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里是为了防止，有用户会在docker和本机同时编译所以在这里以系统分文件夹, 比如在macos同时使用docker和本机

目录结构

├── ...
└── lapack/
    ├── .../
    |── Darwin/
    |     ├── ...
    |     └──xxx.gz
    └── Linux/
         ├── ...
         └──xxx.gz

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a great idea, and we didn't think about that before

set(LAPACK_INSTALL_DIR ${THIRD_PARTY_PATH}/install/lapack)
set(LAPACK_LIB_DIR ${LAPACK_INSTALL_DIR}/lib)

# Note(zhouwei): lapack need fortan compiler which many machines don't have, so use precompiled library.
# use lapack tag v3.10.0 on 06/28/2021 https://github.com/Reference-LAPACK/lapack
if(LINUX)
set(LAPACK_VER
"lapack_lnx_v3.10.0.20210628"
"lapack_lnx_v3.10.0.20210628.tar.gz"
CACHE STRING "" FORCE)
set(LAPACK_URL
"https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.tar.gz"
"https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}"
CACHE STRING "" FORCE)
set(LAPACK_URL_MD5 71f8cc8237a8571692f3e07f9a4f25f6)
set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.so.0")
Expand All @@ -36,10 +37,10 @@ if(LINUX)
elseif(WIN32)
# Refer to [lapack-for-windows] http://icl.cs.utk.edu/lapack-for-windows/lapack/#lapacke
set(LAPACK_VER
"lapack_win_v3.10.0.20210628"
"lapack_win_v3.10.0.20210628.zip"
CACHE STRING "" FORCE)
set(LAPACK_URL
"https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.zip"
"https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}"
CACHE STRING "" FORCE)
set(LAPACK_URL_MD5 590d080392dcd5abbd5dca767a50b63a)
set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath-0.dll")
Expand All @@ -49,10 +50,10 @@ elseif(WIN32)
set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.dll")
else()
set(LAPACK_VER
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个如果把版本改成带后缀的文件名，那最好就换个变量名，比如LAPACK_FILE之类

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

"lapack_mac_v3.10.0.20210628"
"lapack_mac_v3.10.0.20210628.tar.gz"
CACHE STRING "" FORCE)
set(LAPACK_URL
"https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.tar.gz"
"https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}"
CACHE STRING "" FORCE)
set(LAPACK_URL_MD5 427aecf8dee8523de3566ca8e47944d7)
set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.0.dylib")
Expand All @@ -62,18 +63,53 @@ else()
set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.3.dylib")
endif()

function(download_lapack)
message(
STATUS "Downloading ${LAPACK_URL} to ${LAPACK_DOWNLOAD_DIR}/${LAPACK_VER}")
# NOTE: If the version is updated, consider emptying the folder; maybe add timeout
file(
DOWNLOAD ${LAPACK_URL} ${LAPACK_DOWNLOAD_DIR}/${LAPACK_VER}
EXPECTED_MD5 ${LAPACK_URL_MD5}
STATUS ERR)
if(ERR EQUAL 0)
message(STATUS "Download ${LAPACK_VER} success")
else()
message(
FATAL_ERROR
"Download failed, error: ${ERR}\n You can try downloading ${LAPACK_VER} again"
)
endif()
endfunction()

find_file(
LOCAL_LAPACK_LIB_ZIP
NAMES ${LAPACK_VER}
PATHS ${LAPACK_DOWNLOAD_DIR}
NO_DEFAULT_PATH)

# Download and check lapack.
if(LOCAL_LAPACK_LIB_ZIP)
file(MD5 ${LAPACK_DOWNLOAD_DIR}/${LAPACK_VER} LAPACK_MD5)
if(NOT LAPACK_MD5 EQUAL LAPACK_URL_MD5)
download_lapack()
endif()
risemeup1 marked this conversation as resolved.
Show resolved Hide resolved
else()
download_lapack()
endif()

ExternalProject_Add(
extern_lapack
${EXTERNAL_PROJECT_LOG_ARGS}
URL ${LAPACK_URL}
URL ${LAPACK_DOWNLOAD_DIR}/${LAPACK_VER}
URL_MD5 ${LAPACK_URL_MD5}
DOWNLOAD_DIR ${LAPACK_DOWNLOAD_DIR}
SOURCE_DIR ${LAPACK_LIB_DIR}
PREFIX ${LAPACK_PREFIX_DIR}
DOWNLOAD_NO_PROGRESS 1
PATCH_COMMAND ""
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${LAPACK_SOURCE_DIR}
${LAPACK_LIB_DIR}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个直接删除了没有影响吗？

Copy link
Member Author

@gouzil gouzil Jun 6, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里没问题，因为指定了SOURCE_DIR直接就解压到原先的LAPACK_LIB_DIR里面了

INSTALL_COMMAND ""
BUILD_BYPRODUCTS ${BLAS_LIB}
BUILD_BYPRODUCTS ${LAPACK_LIB})
23 changes: 16 additions & 7 deletions cmake/external/libmct.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,17 @@ if((NOT DEFINED LIBMCT_VER) OR (NOT DEFINED LIBMCT_URL))
set(LIBMCT_NAME
"libmct"
CACHE STRING "" FORCE)
set(LIBMCT_DOWNLOAD_FILENAME
"${LIBMCT_NAME}.tar.gz"
CACHE STRING "" FORCE)
set(LIBMCT_URL
"https://pslib.bj.bcebos.com/libmct/libmct.tar.gz"
"https://pslib.bj.bcebos.com/libmct/${LIBMCT_DOWNLOAD_FILENAME}"
CACHE STRING "" FORCE)
endif()
message(STATUS "LIBMCT_NAME: ${LIBMCT_NAME}, LIBMCT_URL: ${LIBMCT_URL}")
set(LIBMCT_PREFIX_DIR "${THIRD_PARTY_PATH}/libmct")
set(LIBMCT_DOWNLOAD_DIR "${LIBMCT_PREFIX_DIR}/src/${LIBMCT_PROJECT}")
set(LIBMCT_DOWNLOAD_DIR
${PADDLE_SOURCE_DIR}/third_party/libmct/${CMAKE_SYSTEM_NAME})
set(LIBMCT_DST_DIR "libmct")
set(LIBMCT_INSTALL_ROOT "${THIRD_PARTY_PATH}/install")
set(LIBMCT_INSTALL_DIR ${LIBMCT_INSTALL_ROOT}/${LIBMCT_DST_DIR})
Expand All @@ -39,22 +43,27 @@ set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${LIBMCT_ROOT}/lib")

include_directories(${LIBMCT_INC_DIR})

file(
DOWNLOAD ${LIBMCT_URL} ${LIBMCT_DOWNLOAD_DIR}/${LIBMCT_DOWNLOAD_FILENAME}
TLS_VERIFY OFF
STATUS ERR)

file(
WRITE ${LIBMCT_DOWNLOAD_DIR}/CMakeLists.txt
"PROJECT(LIBMCT)\n" "cmake_minimum_required(VERSION 3.0)\n"
"install(DIRECTORY ${LIBMCT_NAME}/include ${LIBMCT_NAME}/lib \n"
"install(DIRECTORY ./include ./lib \n"
" DESTINATION ${LIBMCT_DST_DIR})\n")

ExternalProject_Add(
${LIBMCT_PROJECT}
${EXTERNAL_PROJECT_LOG_ARGS}
URL ${LIBMCT_DOWNLOAD_DIR}/${LIBMCT_DOWNLOAD_FILENAME}
PREFIX ${LIBMCT_PREFIX_DIR}
DOWNLOAD_DIR ${LIBMCT_DOWNLOAD_DIR}
DOWNLOAD_COMMAND
wget --no-check-certificate ${LIBMCT_URL} -c -q -O ${LIBMCT_NAME}.tar.gz &&
tar --no-same-owner -zxvf ${LIBMCT_NAME}.tar.gz
DOWNLOAD_NO_PROGRESS 1
SOURCE_DIR ${LIBMCT_INSTALL_DIR}
UPDATE_COMMAND ""
COMMAND ${CMAKE_COMMAND} -E copy ${LIBMCT_DOWNLOAD_DIR}/CMakeLists.txt
${LIBMCT_INSTALL_DIR}
Comment on lines +65 to +66
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里用拷贝是因为当make -j时会清空build/third_parth/install/libmct文件夹

CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBMCT_INSTALL_ROOT}
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${LIBMCT_INSTALL_ROOT}
Expand Down
55 changes: 44 additions & 11 deletions cmake/external/mklml.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@ set(MKLML_INSTALL_DIR ${THIRD_PARTY_PATH}/install/mklml)
set(MKLML_INC_DIR ${MKLML_INSTALL_DIR}/include)
set(MKLML_LIB_DIR ${MKLML_INSTALL_DIR}/lib)
set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${MKLML_LIB_DIR}")
set(MKLML_DOWNLOAD_DIR
${PADDLE_SOURCE_DIR}/third_party/mklml/${CMAKE_SYSTEM_NAME})

if(WIN32)
set(MKLML_VER
"mklml_win_2019.0.5.20190502"
"mklml_win_2019.0.5.20190502.zip"
CACHE STRING "" FORCE)
set(MKLML_URL
"https://paddlepaddledeps.bj.bcebos.com/${MKLML_VER}.zip"
"https://paddlepaddledeps.bj.bcebos.com/${MKLML_VER}"
CACHE STRING "" FORCE)
set(MKLML_URL_MD5 ff8c5237570f03eea37377ccfc95a08a)
set(MKLML_LIB ${MKLML_LIB_DIR}/mklml.lib)
Expand All @@ -35,10 +37,10 @@ else()
# Now enable csrmm function in mklml library temporarily,
# it will be updated as offical version later.
set(MKLML_VER
"csrmm_mklml_lnx_2019.0.5"
"csrmm_mklml_lnx_2019.0.5.tgz"
CACHE STRING "" FORCE)
set(MKLML_URL
"http://paddlepaddledeps.bj.bcebos.com/${MKLML_VER}.tgz"
"http://paddlepaddledeps.bj.bcebos.com/${MKLML_VER}"
CACHE STRING "" FORCE)
set(MKLML_URL_MD5 bc6a7faea6a2a9ad31752386f3ae87da)
set(MKLML_LIB ${MKLML_LIB_DIR}/libmklml_intel.so)
Expand All @@ -50,7 +52,40 @@ endif()
set(MKLML_PROJECT "extern_mklml")
message(STATUS "MKLML_VER: ${MKLML_VER}, MKLML_URL: ${MKLML_URL}")
set(MKLML_PREFIX_DIR ${THIRD_PARTY_PATH}/mklml)
set(MKLML_SOURCE_DIR ${THIRD_PARTY_PATH}/mklml/src/extern_mklml)

function(download_mklml)
message(
STATUS "Downloading ${MKLML_URL} to ${MKLML_DOWNLOAD_DIR}/${MKLML_VER}")
# NOTE: If the version is updated, consider emptying the folder; maybe add timeout
file(
DOWNLOAD ${MKLML_URL} ${MKLML_DOWNLOAD_DIR}/${MKLML_VER}
EXPECTED_MD5 ${MKLML_URL_MD5}
STATUS ERR)
if(ERR EQUAL 0)
message(STATUS "Download ${MKLML_VER} success")
else()
message(
FATAL_ERROR
"Download failed, error: ${ERR}\n You can try downloading ${MKLML_VER} again"
)
endif()
endfunction()

find_file(
LOCAL_MKLML_LIB_ZIP
NAMES ${MKLML_VER}
PATHS ${MKLML_DOWNLOAD_DIR}
NO_DEFAULT_PATH)

# Download and check mklml.
if(LOCAL_MKLML_LIB_ZIP)
file(MD5 ${MKLML_DOWNLOAD_DIR}/${MKLML_VER} MKLML_MD5)
if(NOT MKLML_MD5 EQUAL MKLML_URL_MD5)
download_mklml()
endif()
else()
download_mklml()
endif()

# Ninja Generator can not establish the correct dependency relationship
# between the imported library with target, the product file
Expand All @@ -60,17 +95,15 @@ set(MKLML_SOURCE_DIR ${THIRD_PARTY_PATH}/mklml/src/extern_mklml)
ExternalProject_Add(
${MKLML_PROJECT}
${EXTERNAL_PROJECT_LOG_ARGS}
URL ${MKLML_URL}
URL ${MKLML_DOWNLOAD_DIR}/${MKLML_VER}
URL_MD5 ${MKLML_URL_MD5}
DOWNLOAD_DIR ${MKLML_DOWNLOAD_DIR}
SOURCE_DIR ${MKLML_INSTALL_DIR}
PREFIX ${MKLML_PREFIX_DIR}
DOWNLOAD_NO_PROGRESS 1
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
UPDATE_COMMAND ""
INSTALL_COMMAND
${CMAKE_COMMAND} -E copy_directory ${MKLML_SOURCE_DIR}/include
${MKLML_INC_DIR} && ${CMAKE_COMMAND} -E copy_directory
${MKLML_SOURCE_DIR}/lib ${MKLML_LIB_DIR}
INSTALL_COMMAND ""
BUILD_BYPRODUCTS ${MKLML_LIB}
BUILD_BYPRODUCTS ${MKLML_IOMP_LIB})

Expand Down