merge develop

PaddlePaddle · Apr 27, 2023 · fff8e6e · fff8e6e
2 parents b832793 + c50f5fa
commit fff8e6e
Show file tree

Hide file tree

Showing 458 changed files with 8,410 additions and 4,405 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -246,6 +246,7 @@ option(WITH_DISTRIBUTE "Compile with distributed support" OFF)
 option(WITH_BRPC_RDMA "Use brpc rdma as the rpc protocal" OFF)
 option(ON_INFER "Turn on inference optimization and inference-lib generation"
        ON)
+option(WITH_CPP_DIST "Install PaddlePaddle C++ distribution" OFF)
 ################################ Internal Configurations #######################################
 option(WITH_NV_JETSON "Compile PaddlePaddle with NV JETSON" OFF)
 option(WITH_PROFILER "Compile PaddlePaddle with GPU profiler and gperftools"
@@ -662,6 +663,21 @@ if(WITH_STRIP)
   endif()
 endif()
 
+if(WITH_CPP_DIST)
+  # TODO(huangjiyi): Separate installing C++ distribution from python package
+  # installation and support for installing C++ distribution on more platforms.
+  if(NOT LINUX OR NOT WITH_PYTHON)
+    set(WITH_CPP_DIST
+        OFF
+        CACHE
+          STRING
+          "Currently C++ Distribution Generation is only available on Linux and compiling WITH_PYTHON=ON."
+          FORCE)
+  else()
+    include(paddle_lib)
+  endif()
+endif()
+
 add_subdirectory(paddle)
 if(WITH_PYTHON)
   add_subdirectory(python)

diff --git a/cmake/PaddleConfig.cmake.in b/cmake/PaddleConfig.cmake.in
@@ -0,0 +1,33 @@
+# Paddle CMake configuration file
+# -------
+#
+# Finds the Paddle library
+#
+# This will define the following variables:
+#
+#   PADDLE_FOUND        -- True if the system has the Paddle library
+#   PADDLE_INCLUDE_DIRS -- The include directories for Paddle
+#   PADDLE_LIBRARIES    -- Libraries to link against
+
+get_filename_component(PADDLE_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_FILE}/../.." ABSOLUTE)
+
+# include directories
+set(PADDLE_INCLUDE_DIRS 
+    ${PADDLE_INSTALL_PREFIX}/include
+    ${PADDLE_INSTALL_PREFIX}/include/third_party
+)
+
+# Library dependencies.
+set(PADDLE_LIBRARIES_DIRS ${PADDLE_INSTALL_PREFIX}/lib)
+link_directories(${PADDLE_LIBRARIES_DIRS})
+
+file(GLOB PADDLE_LIBRARIES ${PADDLE_LIBRARIES_DIRS}/lib*)
+
+find_package(PythonLibs @PY_VERSION@ REQUIRED)
+list(APPEND PADDLE_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS})
+list(APPEND PADDLE_LIBRARIES ${PYTHON_LIBRARIES})
+
+if(@WITH_GPU@)
+    find_package(CUDA @CUDA_VERSION@ REQUIRED)
+    list(APPEND PADDLE_LIBRARIES ${CUDA_LIBRARIES})
+endif()
diff --git a/cmake/external/dlpack.cmake b/cmake/external/dlpack.cmake
@@ -17,6 +17,7 @@ include(ExternalProject)
 set(DLPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/dlpack)
 
 set(DLPACK_REPOSITORY ${GIT_URL}/dmlc/dlpack.git)
+
 set(DLPACK_TAG v0.4)
 
 set(DLPACK_INCLUDE_DIR ${THIRD_PARTY_PATH}/dlpack/src/extern_dlpack/include)

diff --git a/cmake/external/eigen.cmake b/cmake/external/eigen.cmake
@@ -51,9 +51,10 @@ if(CMAKE_COMPILER_IS_GNUCC)
   if(GCC_VERSION GREATER_EQUAL "12.0")
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Complex.h.patch
          complex_header)
+    # See: [Why calling some `git` commands before `patch`?]
     set(EIGEN_PATCH_COMMAND
-        patch -d ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ <
-        ${complex_header})
+        git checkout -- . && git checkout ${EIGEN_TAG} && patch -Nd
+        ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
   endif()
 endif()
 

diff --git a/cmake/external/gloo.cmake b/cmake/external/gloo.cmake
@@ -37,8 +37,9 @@ if(WITH_GPU)
                                                   VERSION_GREATER 12.0)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/device.cc.patch
          native_dst)
-    set(GLOO_PATCH_COMMAND patch -d ${GLOO_SOURCE_DIR}/gloo/transport/tcp <
-                           ${native_dst})
+    set(GLOO_PATCH_COMMAND
+        git checkout -- . && git checkout ${GLOO_TAG} &&patch -Nd
+        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
   endif()
 endif()
 
@@ -54,9 +55,11 @@ if(CMAKE_COMPILER_IS_GNUCC)
          native_dst)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/types.h.patch
          types_header)
+    # See: [Why calling some `git` commands before `patch`?]
     set(GLOO_PATCH_COMMAND
-        patch -d ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch
-        -d ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
+        git checkout -- . && git checkout ${GLOO_TAG} && patch -Nd
+        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
+        ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
   endif()
 endif()
 include_directories(${GLOO_INCLUDE_DIR})

diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake
@@ -66,8 +66,10 @@ endif()
 if(NOT WIN32 AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0)
   file(TO_NATIVE_PATH
        ${PADDLE_SOURCE_DIR}/patches/gtest/gtest-death-test.cc.patch native_src)
-  set(GTEST_PATCH_COMMAND patch -d ${GTEST_SOURCE_DIR}/googletest/src <
-                          ${native_src})
+  # See: [Why calling some `git` commands before `patch`?]
+  set(GTEST_PATCH_COMMAND
+      git checkout -- . && git checkout ${GTEST_TAG} && patch -Nd
+      ${GTEST_SOURCE_DIR}/googletest/src < ${native_src})
 endif()
 if(WIN32)
   ExternalProject_Add(

diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake
@@ -25,8 +25,13 @@ set(PYBIND_PATCH_COMMAND "")
 if(NOT WIN32)
   file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/pybind/cast.h.patch
        native_dst)
-  set(PYBIND_PATCH_COMMAND patch -d ${PYBIND_INCLUDE_DIR}/pybind11 <
-                           ${native_dst})
+  # Note: [Why calling some `git` commands before `patch`?]
+  # Paddle's CI uses cache to accelarate the make process. However, error might raise when patch codes in two scenarios:
+  # 1. Patch to the wrong version: the tag version of CI's cache falls behind PYBIND_TAG, use `git checkout ${PYBIND_TAG}` to solve this.
+  # 2. Patch twice: the tag version of cache == PYBIND_TAG, but patch has already applied to cache.
+  set(PYBIND_PATCH_COMMAND
+      git checkout -- . && git checkout ${PYBIND_TAG} && patch -Nd
+      ${PYBIND_INCLUDE_DIR}/pybind11 < ${native_dst})
 endif()
 
 ExternalProject_Add(

diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake
@@ -12,11 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# make package for paddle fluid shared and static library
-set(PADDLE_INSTALL_DIR
-    "${CMAKE_BINARY_DIR}/paddle_install_dir"
-    CACHE STRING "A path setting paddle shared and static libraries")
-
+# make package for paddle inference shared and static library
 set(PADDLE_INFERENCE_INSTALL_DIR
     "${CMAKE_BINARY_DIR}/paddle_inference_install_dir"
     CACHE STRING "A path setting paddle inference shared and static libraries")
@@ -214,7 +210,7 @@ endfunction()
 # inference library for only inference
 set(inference_lib_deps third_party paddle_inference paddle_inference_c
                        paddle_inference_shared paddle_inference_c_shared)
-add_custom_target(inference_lib_dist DEPENDS ${inference_lib_deps})
+add_custom_target(inference_lib_dist ALL DEPENDS ${inference_lib_deps})
 
 set(dst_dir "${PADDLE_INFERENCE_INSTALL_DIR}/third_party/threadpool")
 copy(
@@ -387,158 +383,5 @@ if(WITH_STRIP AND NOT WIN32)
   )
 endif()
 
-# fluid library for both train and inference
-set(fluid_lib_deps inference_lib_dist)
-add_custom_target(fluid_lib_dist ALL DEPENDS ${fluid_lib_deps})
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/paddle/fluid")
-set(module "inference")
-if(WIN32)
-  copy(
-    fluid_lib_dist
-    SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/api/paddle_*.h
-         ${paddle_inference_lib}
-    DSTS ${dst_dir}/${module} ${dst_dir}/${module} ${dst_dir}/${module}
-         ${dst_dir}/${module})
-else()
-  copy(
-    fluid_lib_dist
-    SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/api/paddle_*.h
-         ${paddle_inference_lib}
-    DSTS ${dst_dir}/${module} ${dst_dir}/${module} ${dst_dir}/${module})
-endif()
-
-set(module "framework")
-set(framework_lib_deps framework_proto data_feed_proto trainer_desc_proto)
-add_dependencies(fluid_lib_dist ${framework_lib_deps})
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/*.h
-       ${src_dir}/${module}/details/*.h
-       ${PADDLE_BINARY_DIR}/paddle/fluid/framework/trainer_desc.pb.h
-       ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.h
-       ${PADDLE_BINARY_DIR}/paddle/fluid/framework/data_feed.pb.h
-       ${src_dir}/${module}/ir/memory_optimize_pass/*.h
-       ${src_dir}/${module}/ir/*.h
-       ${src_dir}/${module}/fleet/*.h
-  DSTS ${dst_dir}/${module}
-       ${dst_dir}/${module}/details
-       ${dst_dir}/${module}
-       ${dst_dir}/${module}
-       ${dst_dir}/${module}
-       ${dst_dir}/${module}/ir/memory_optimize_pass
-       ${dst_dir}/${module}/ir
-       ${dst_dir}/${module}/fleet)
-
-set(module "operators")
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/reader/blocking_queue.h
-  DSTS ${dst_dir}/${module}/reader/)
-
-set(module "memory")
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/allocation/*.h
-  DSTS ${dst_dir}/${module}/allocation)
-
-set(module "platform")
-set(platform_lib_deps phi_profiler_proto errors)
-if(WITH_GPU)
-  set(platform_lib_deps ${platform_lib_deps} external_error_proto)
-endif()
-
-add_dependencies(fluid_lib_dist ${platform_lib_deps})
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/dynload/*.h
-       ${PADDLE_BINARY_DIR}/paddle/phi/api/profiler/*.pb.h
-  DSTS ${dst_dir}/${module} ${dst_dir}/${module}/dynload ${dst_dir}/${module})
-
-set(module "string")
-copy(
-  fluid_lib_dist
-  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/${module}/*.h
-       ${PADDLE_SOURCE_DIR}/paddle/utils/${module}/tinyformat/*.h
-  DSTS ${dst_dir}/${module} ${dst_dir}/${module}/tinyformat)
-
-set(module "imperative")
-copy(
-  fluid_lib_dist
-  SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/jit/*.h
-  DSTS ${dst_dir}/${module} ${dst_dir}/${module}/jit)
-
-set(module "pybind")
-copy(
-  fluid_lib_dist
-  SRCS ${CMAKE_CURRENT_BINARY_DIR}/paddle/fluid/${module}/pybind.h
-  DSTS ${dst_dir}/${module})
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/third_party/eigen3")
-copy(
-  inference_lib_dist
-  SRCS ${EIGEN_INCLUDE_DIR}/Eigen/Core ${EIGEN_INCLUDE_DIR}/Eigen/src
-       ${EIGEN_INCLUDE_DIR}/unsupported/Eigen
-  DSTS ${dst_dir}/Eigen ${dst_dir}/Eigen ${dst_dir}/unsupported)
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/third_party/dlpack")
-copy(
-  inference_lib_dist
-  SRCS ${DLPACK_INCLUDE_DIR}/dlpack
-  DSTS ${dst_dir})
-
-set(dst_dir "${PADDLE_INSTALL_DIR}/third_party/install/zlib")
-copy(
-  inference_lib_dist
-  SRCS ${ZLIB_INCLUDE_DIR} ${ZLIB_LIBRARIES}
-  DSTS ${dst_dir} ${dst_dir}/lib)
-
-# CMakeCache Info
-copy(
-  fluid_lib_dist
-  SRCS ${PADDLE_INFERENCE_INSTALL_DIR}/third_party
-       ${CMAKE_CURRENT_BINARY_DIR}/CMakeCache.txt
-  DSTS ${PADDLE_INSTALL_DIR} ${PADDLE_INSTALL_DIR})
-
-# paddle fluid version
-function(version version_file)
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} log --pretty=format:%H -1
-    WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}
-    OUTPUT_VARIABLE PADDLE_GIT_COMMIT)
-  file(
-    WRITE ${version_file}
-    "GIT COMMIT ID: ${PADDLE_GIT_COMMIT}\n" "WITH_MKL: ${WITH_MKL}\n"
-    "WITH_MKLDNN: ${WITH_MKLDNN}\n" "WITH_GPU: ${WITH_GPU}\n"
-    "WITH_ROCM: ${WITH_ROCM}\n" "WITH_IPU: ${WITH_IPU}\n")
-  if(WITH_GPU)
-    file(APPEND ${version_file}
-         "CUDA version: ${CUDA_VERSION}\n"
-         "CUDNN version: v${CUDNN_MAJOR_VERSION}.${CUDNN_MINOR_VERSION}\n")
-  endif()
-  if(WITH_ROCM)
-    file(APPEND ${version_file}
-         "HIP version: v${HIP_MAJOR_VERSION}.${HIP_MINOR_VERSION}\n"
-         "MIOpen version: v${MIOPEN_MAJOR_VERSION}.${MIOPEN_MINOR_VERSION}\n")
-  endif()
-  if(WITH_IPU)
-    file(APPEND ${version_file} "PopART version: ${POPART_VERSION}\n")
-  endif()
-  file(APPEND ${version_file}
-       "CXX compiler version: ${CMAKE_CXX_COMPILER_VERSION}\n")
-  if(TENSORRT_FOUND)
-    file(
-      APPEND ${version_file}
-      "WITH_TENSORRT: ${TENSORRT_FOUND}\n"
-      "TensorRT version: v${TENSORRT_MAJOR_VERSION}.${TENSORRT_MINOR_VERSION}.${TENSORRT_PATCH_VERSION}.${TENSORRT_BUILD_VERSION}\n"
-    )
-  endif()
-  if(WITH_LITE)
-    file(APPEND ${version_file} "WITH_LITE: ${WITH_LITE}\n"
-                                "LITE_GIT_TAG: ${LITE_GIT_TAG}\n")
-  endif()
-
-endfunction()
-version(${PADDLE_INSTALL_DIR}/version.txt)
 version(${PADDLE_INFERENCE_INSTALL_DIR}/version.txt)
 version(${PADDLE_INFERENCE_C_INSTALL_DIR}/version.txt)
diff --git a/cmake/operators.cmake b/cmake/operators.cmake
@@ -51,6 +51,58 @@ function(find_phi_register FILENAME ADD_PATH PATTERN)
   endif()
 endfunction()
 
+# Just for those gpu kernels locating at "fluid/operators/", such as 'class_center_sample_op.cu'.
+# Add other file modes if need in the future.
+function(register_cu_kernel TARGET)
+  set(options "")
+  set(oneValueArgs "")
+  set(multiValueArgs SRCS DEPS)
+  cmake_parse_arguments(register_cu_kernel "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN})
+
+  set(cu_srcs)
+  set(op_common_deps operator op_registry math_function layer
+                     common_infer_shape_functions)
+  foreach(cu_src ${register_cu_kernel_SRCS})
+    if(${cu_src} MATCHES ".*\\.cu$")
+      list(APPEND cu_srcs ${cu_src})
+    endif()
+  endforeach()
+  list(LENGTH cu_srcs cu_srcs_len)
+  if(${cu_srcs_len} EQUAL 0)
+    message(
+      FATAL_ERROR
+        "The GPU kernel file of ${TARGET} should contains at least one .cu file"
+    )
+  endif()
+  if(WITH_GPU)
+    nv_library(
+      ${TARGET}
+      SRCS ${cu_srcs}
+      DEPS ${op_library_DEPS} ${op_common_deps})
+  elseif(WITH_ROCM)
+    hip_library(
+      ${TARGET}
+      SRCS ${cu_srcs}
+      DEPS ${op_library_DEPS} ${op_common_deps})
+  endif()
+  set(OP_LIBRARY
+      ${TARGET} ${OP_LIBRARY}
+      CACHE INTERNAL "op libs")
+  foreach(cu_src ${cu_srcs})
+    set(op_name "")
+    # Add PHI Kernel Registry Message
+    find_phi_register(${cu_src} ${pybind_file} "PD_REGISTER_KERNEL")
+    find_phi_register(${cu_src} ${pybind_file} "PD_REGISTER_STRUCT_KERNEL")
+    find_phi_register(${cu_src} ${pybind_file}
+                      "PD_REGISTER_KERNEL_FOR_ALL_DTYPE")
+    find_register(${cu_src} "REGISTER_OP_CUDA_KERNEL" op_name)
+    if(NOT ${op_name} EQUAL "")
+      file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, CUDA);\n")
+    endif()
+  endforeach()
+endfunction()
+
 function(op_library TARGET)
   # op_library is a function to create op library. The interface is same as
   # cc_library. But it handle split GPU/CPU code and link some common library
@@ -470,6 +522,7 @@ function(op_library TARGET)
       foreach(xpu_src ${xpu_cc_srcs})
         set(op_name "")
         find_register(${xpu_src} "REGISTER_OP_XPU_KERNEL" op_name)
+        find_phi_register(${xpu_src} ${pybind_file} "PD_REGISTER_STRUCT_KERNEL")
         if(NOT ${op_name} EQUAL "")
           file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, XPU);\n")
           set(pybind_flag 1)