diff --git a/cmake/external/lapack.cmake b/cmake/external/lapack.cmake
index 43305223fe280..78c2659a58504 100644
--- a/cmake/external/lapack.cmake
+++ b/cmake/external/lapack.cmake
@@ -15,18 +15,19 @@
 include(ExternalProject)
 
 set(LAPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/lapack)
-set(LAPACK_SOURCE_DIR ${THIRD_PARTY_PATH}/lapack/src/extern_lapack)
+set(LAPACK_DOWNLOAD_DIR
+    ${PADDLE_SOURCE_DIR}/third_party/lapack/${CMAKE_SYSTEM_NAME})
 set(LAPACK_INSTALL_DIR ${THIRD_PARTY_PATH}/install/lapack)
 set(LAPACK_LIB_DIR ${LAPACK_INSTALL_DIR}/lib)
 
 # Note(zhouwei): lapack need fortan compiler which many machines don't have, so use precompiled library.
 # use lapack tag v3.10.0 on 06/28/2021 https://github.com/Reference-LAPACK/lapack
 if(LINUX)
-  set(LAPACK_VER
-      "lapack_lnx_v3.10.0.20210628"
+  set(LAPACK_FILE
+      "lapack_lnx_v3.10.0.20210628.tar.gz"
       CACHE STRING "" FORCE)
   set(LAPACK_URL
-      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.tar.gz"
+      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}"
       CACHE STRING "" FORCE)
   set(LAPACK_URL_MD5 71f8cc8237a8571692f3e07f9a4f25f6)
   set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.so.0")
@@ -35,11 +36,11 @@ if(LINUX)
   set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.so.3")
 elseif(WIN32)
   # Refer to [lapack-for-windows] http://icl.cs.utk.edu/lapack-for-windows/lapack/#lapacke
-  set(LAPACK_VER
-      "lapack_win_v3.10.0.20210628"
+  set(LAPACK_FILE
+      "lapack_win_v3.10.0.20210628.zip"
       CACHE STRING "" FORCE)
   set(LAPACK_URL
-      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.zip"
+      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}"
       CACHE STRING "" FORCE)
   set(LAPACK_URL_MD5 590d080392dcd5abbd5dca767a50b63a)
   set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath-0.dll")
@@ -48,11 +49,11 @@ elseif(WIN32)
   set(BLAS_LIB "${LAPACK_LIB_DIR}/libblas.dll")
   set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.dll")
 else()
-  set(LAPACK_VER
-      "lapack_mac_v3.10.0.20210628"
+  set(LAPACK_FILE
+      "lapack_mac_v3.10.0.20210628.tar.gz"
       CACHE STRING "" FORCE)
   set(LAPACK_URL
-      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.tar.gz"
+      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}"
       CACHE STRING "" FORCE)
   set(LAPACK_URL_MD5 427aecf8dee8523de3566ca8e47944d7)
   set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.0.dylib")
@@ -62,18 +63,53 @@ else()
   set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.3.dylib")
 endif()
 
+function(download_lapack)
+  message(
+    STATUS "Downloading ${LAPACK_URL} to ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE}")
+  # NOTE: If the version is updated, consider emptying the folder; maybe add timeout
+  file(
+    DOWNLOAD ${LAPACK_URL} ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE}
+    EXPECTED_MD5 ${LAPACK_URL_MD5}
+    STATUS ERR)
+  if(ERR EQUAL 0)
+    message(STATUS "Download ${LAPACK_FILE} success")
+  else()
+    message(
+      FATAL_ERROR
+        "Download failed, error: ${ERR}\n You can try downloading ${LAPACK_FILE} again"
+    )
+  endif()
+endfunction()
+
+find_file(
+  LOCAL_LAPACK_LIB_ZIP
+  NAMES ${LAPACK_FILE}
+  PATHS ${LAPACK_DOWNLOAD_DIR}
+  NO_DEFAULT_PATH)
+
+# Download and check lapack.
+if(LOCAL_LAPACK_LIB_ZIP)
+  file(MD5 ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE} LAPACK_MD5)
+  if(NOT LAPACK_MD5 EQUAL LAPACK_URL_MD5)
+    download_lapack()
+  endif()
+else()
+  download_lapack()
+endif()
+
 ExternalProject_Add(
   extern_lapack
   ${EXTERNAL_PROJECT_LOG_ARGS}
-  URL ${LAPACK_URL}
+  URL ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE}
   URL_MD5 ${LAPACK_URL_MD5}
+  DOWNLOAD_DIR ${LAPACK_DOWNLOAD_DIR}
+  SOURCE_DIR ${LAPACK_LIB_DIR}
   PREFIX ${LAPACK_PREFIX_DIR}
   DOWNLOAD_NO_PROGRESS 1
   PATCH_COMMAND ""
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""
   BUILD_COMMAND ""
-  INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${LAPACK_SOURCE_DIR}
-                  ${LAPACK_LIB_DIR}
+  INSTALL_COMMAND ""
   BUILD_BYPRODUCTS ${BLAS_LIB}
   BUILD_BYPRODUCTS ${LAPACK_LIB})
diff --git a/cmake/external/libmct.cmake b/cmake/external/libmct.cmake
index 28bf083f7791e..15ee59642094a 100644
--- a/cmake/external/libmct.cmake
+++ b/cmake/external/libmct.cmake
@@ -23,13 +23,17 @@ if((NOT DEFINED LIBMCT_VER) OR (NOT DEFINED LIBMCT_URL))
   set(LIBMCT_NAME
       "libmct"
       CACHE STRING "" FORCE)
+  set(LIBMCT_DOWNLOAD_FILE
+      "${LIBMCT_NAME}.tar.gz"
+      CACHE STRING "" FORCE)
   set(LIBMCT_URL
-      "https://pslib.bj.bcebos.com/libmct/libmct.tar.gz"
+      "https://pslib.bj.bcebos.com/libmct/${LIBMCT_DOWNLOAD_FILE}"
       CACHE STRING "" FORCE)
 endif()
 message(STATUS "LIBMCT_NAME: ${LIBMCT_NAME}, LIBMCT_URL: ${LIBMCT_URL}")
 set(LIBMCT_PREFIX_DIR "${THIRD_PARTY_PATH}/libmct")
-set(LIBMCT_DOWNLOAD_DIR "${LIBMCT_PREFIX_DIR}/src/${LIBMCT_PROJECT}")
+set(LIBMCT_DOWNLOAD_DIR
+    ${PADDLE_SOURCE_DIR}/third_party/libmct/${CMAKE_SYSTEM_NAME})
 set(LIBMCT_DST_DIR "libmct")
 set(LIBMCT_INSTALL_ROOT "${THIRD_PARTY_PATH}/install")
 set(LIBMCT_INSTALL_DIR ${LIBMCT_INSTALL_ROOT}/${LIBMCT_DST_DIR})
@@ -39,22 +43,27 @@ set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${LIBMCT_ROOT}/lib")
 
 include_directories(${LIBMCT_INC_DIR})
 
+file(
+  DOWNLOAD ${LIBMCT_URL} ${LIBMCT_DOWNLOAD_DIR}/${LIBMCT_DOWNLOAD_FILE}
+  TLS_VERIFY OFF
+  STATUS ERR)
+
 file(
   WRITE ${LIBMCT_DOWNLOAD_DIR}/CMakeLists.txt
   "PROJECT(LIBMCT)\n" "cmake_minimum_required(VERSION 3.0)\n"
-  "install(DIRECTORY ${LIBMCT_NAME}/include ${LIBMCT_NAME}/lib \n"
+  "install(DIRECTORY ./include ./lib \n"
   "        DESTINATION ${LIBMCT_DST_DIR})\n")
 
 ExternalProject_Add(
   ${LIBMCT_PROJECT}
   ${EXTERNAL_PROJECT_LOG_ARGS}
+  URL ${LIBMCT_DOWNLOAD_DIR}/${LIBMCT_DOWNLOAD_FILE}
   PREFIX ${LIBMCT_PREFIX_DIR}
   DOWNLOAD_DIR ${LIBMCT_DOWNLOAD_DIR}
-  DOWNLOAD_COMMAND
-    wget --no-check-certificate ${LIBMCT_URL} -c -q -O ${LIBMCT_NAME}.tar.gz &&
-    tar --no-same-owner -zxvf ${LIBMCT_NAME}.tar.gz
-  DOWNLOAD_NO_PROGRESS 1
+  SOURCE_DIR ${LIBMCT_INSTALL_DIR}
   UPDATE_COMMAND ""
+  COMMAND ${CMAKE_COMMAND} -E copy ${LIBMCT_DOWNLOAD_DIR}/CMakeLists.txt
+          ${LIBMCT_INSTALL_DIR}
   CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBMCT_INSTALL_ROOT}
              -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
   CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${LIBMCT_INSTALL_ROOT}
diff --git a/cmake/external/mklml.cmake b/cmake/external/mklml.cmake
index 14a8298790799..801cab62facbb 100644
--- a/cmake/external/mklml.cmake
+++ b/cmake/external/mklml.cmake
@@ -17,13 +17,15 @@ set(MKLML_INSTALL_DIR ${THIRD_PARTY_PATH}/install/mklml)
 set(MKLML_INC_DIR ${MKLML_INSTALL_DIR}/include)
 set(MKLML_LIB_DIR ${MKLML_INSTALL_DIR}/lib)
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${MKLML_LIB_DIR}")
+set(MKLML_DOWNLOAD_DIR
+    ${PADDLE_SOURCE_DIR}/third_party/mklml/${CMAKE_SYSTEM_NAME})
 
 if(WIN32)
-  set(MKLML_VER
-      "mklml_win_2019.0.5.20190502"
+  set(MKLML_FILE
+      "mklml_win_2019.0.5.20190502.zip"
       CACHE STRING "" FORCE)
   set(MKLML_URL
-      "https://paddlepaddledeps.bj.bcebos.com/${MKLML_VER}.zip"
+      "https://paddlepaddledeps.bj.bcebos.com/${MKLML_FILE}"
       CACHE STRING "" FORCE)
   set(MKLML_URL_MD5 ff8c5237570f03eea37377ccfc95a08a)
   set(MKLML_LIB ${MKLML_LIB_DIR}/mklml.lib)
@@ -34,11 +36,11 @@ else()
   #TODO(intel-huying):
   #  Now enable csrmm function in mklml library temporarily,
   #  it will be updated as offical version later.
-  set(MKLML_VER
-      "csrmm_mklml_lnx_2019.0.5"
+  set(MKLML_FILE
+      "csrmm_mklml_lnx_2019.0.5.tgz"
       CACHE STRING "" FORCE)
   set(MKLML_URL
-      "http://paddlepaddledeps.bj.bcebos.com/${MKLML_VER}.tgz"
+      "http://paddlepaddledeps.bj.bcebos.com/${MKLML_FILE}"
       CACHE STRING "" FORCE)
   set(MKLML_URL_MD5 bc6a7faea6a2a9ad31752386f3ae87da)
   set(MKLML_LIB ${MKLML_LIB_DIR}/libmklml_intel.so)
@@ -48,9 +50,42 @@ else()
 endif()
 
 set(MKLML_PROJECT "extern_mklml")
-message(STATUS "MKLML_VER: ${MKLML_VER}, MKLML_URL: ${MKLML_URL}")
+message(STATUS "MKLML_FILE: ${MKLML_FILE}, MKLML_URL: ${MKLML_URL}")
 set(MKLML_PREFIX_DIR ${THIRD_PARTY_PATH}/mklml)
-set(MKLML_SOURCE_DIR ${THIRD_PARTY_PATH}/mklml/src/extern_mklml)
+
+function(download_mklml)
+  message(
+    STATUS "Downloading ${MKLML_URL} to ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE}")
+  # NOTE: If the version is updated, consider emptying the folder; maybe add timeout
+  file(
+    DOWNLOAD ${MKLML_URL} ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE}
+    EXPECTED_MD5 ${MKLML_URL_MD5}
+    STATUS ERR)
+  if(ERR EQUAL 0)
+    message(STATUS "Download ${MKLML_FILE} success")
+  else()
+    message(
+      FATAL_ERROR
+        "Download failed, error: ${ERR}\n You can try downloading ${MKLML_FILE} again"
+    )
+  endif()
+endfunction()
+
+find_file(
+  LOCAL_MKLML_LIB_ZIP
+  NAMES ${MKLML_FILE}
+  PATHS ${MKLML_DOWNLOAD_DIR}
+  NO_DEFAULT_PATH)
+
+# Download and check mklml.
+if(LOCAL_MKLML_LIB_ZIP)
+  file(MD5 ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE} MKLML_MD5)
+  if(NOT MKLML_MD5 EQUAL MKLML_URL_MD5)
+    download_mklml()
+  endif()
+else()
+  download_mklml()
+endif()
 
 # Ninja Generator can not establish the correct dependency relationship
 # between the imported library with target, the product file
@@ -60,17 +95,15 @@ set(MKLML_SOURCE_DIR ${THIRD_PARTY_PATH}/mklml/src/extern_mklml)
 ExternalProject_Add(
   ${MKLML_PROJECT}
   ${EXTERNAL_PROJECT_LOG_ARGS}
-  URL ${MKLML_URL}
+  URL ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE}
   URL_MD5 ${MKLML_URL_MD5}
+  DOWNLOAD_DIR ${MKLML_DOWNLOAD_DIR}
+  SOURCE_DIR ${MKLML_INSTALL_DIR}
   PREFIX ${MKLML_PREFIX_DIR}
-  DOWNLOAD_NO_PROGRESS 1
   CONFIGURE_COMMAND ""
   BUILD_COMMAND ""
   UPDATE_COMMAND ""
-  INSTALL_COMMAND
-    ${CMAKE_COMMAND} -E copy_directory ${MKLML_SOURCE_DIR}/include
-    ${MKLML_INC_DIR} && ${CMAKE_COMMAND} -E copy_directory
-    ${MKLML_SOURCE_DIR}/lib ${MKLML_LIB_DIR}
+  INSTALL_COMMAND ""
   BUILD_BYPRODUCTS ${MKLML_LIB}
   BUILD_BYPRODUCTS ${MKLML_IOMP_LIB})