From 4d6c81dfeff04b1848b19d87cb40ca846bf93839 Mon Sep 17 00:00:00 2001 From: xq114 <1140735506@qq.com> Date: Tue, 30 Mar 2021 22:35:27 +0800 Subject: [PATCH 1/5] add libtorch --- packages/c/cuda/xmake.lua | 27 +++++++ packages/l/libtorch/xmake.lua | 130 ++++++++++++++++++++++++++++++++++ packages/m/mkl/xmake.lua | 10 +++ packages/n/nvtx/xmake.lua | 11 +++ 4 files changed, 178 insertions(+) create mode 100644 packages/c/cuda/xmake.lua create mode 100644 packages/l/libtorch/xmake.lua create mode 100644 packages/m/mkl/xmake.lua create mode 100644 packages/n/nvtx/xmake.lua diff --git a/packages/c/cuda/xmake.lua b/packages/c/cuda/xmake.lua new file mode 100644 index 00000000000..b522e4cb5cf --- /dev/null +++ b/packages/c/cuda/xmake.lua @@ -0,0 +1,27 @@ +package("cuda") + + set_homepage("https://developer.nvidia.com/cuda-zone/") + set_description("CUDA® is a parallel computing platform and programming model developed by NVIDIA for general computing on graphical processing units (GPUs).") + + add_configs("utils", {description = "enabled cuda utilities.", default = {}, type = "table"}) + + on_fetch(function (package, opt) + if opt.system then + import("detect.sdks.find_cuda") + import("lib.detect.find_library") + + local cuda = find_cuda() + if cuda then + package:addenv("PATH", cuda.bindir) + local result = {includedirs = cuda.includedirs, linkdirs = cuda.linkdirs, links = {}} + local utils = package:config("utils") + table.insert(utils, package:config("shared") and "cudart" or "cudart_static") + + for _, util in ipairs(utils) do + if not find_library(util, cuda.linkdirs) then return end + table.insert(result.links, util) + end + return result + end + end + end) diff --git a/packages/l/libtorch/xmake.lua b/packages/l/libtorch/xmake.lua new file mode 100644 index 00000000000..440bc254134 --- /dev/null +++ b/packages/l/libtorch/xmake.lua @@ -0,0 +1,130 @@ +package("libtorch") + + set_homepage("https://pytorch.org/") + set_description("An open source machine learning framework that accelerates the path from research prototyping to production deployment.") + set_license("BSD-3-Clause") + + add_urls("https://github.com/pytorch/pytorch.git") + add_versions("v1.8.0", "37c1f4a7fef115d719104e871d0cf39434aa9d56") + + add_configs("python", {description = "Build python interface.", default = false, type = "boolean"}) + add_configs("ninja", {description = "Use ninja as build tool.", default = false, type = "boolean"}) + + add_deps("cmake") + add_deps("python 3.x", {kind = "binary", system = false}) + add_deps("libuv") + add_deps("cuda", {optional = true, configs = {utils = {"nvrtc", "cudnn", "cufft", "curand", "cublas", "cudart_static"}}}) + add_deps("nvtx", "mkl", {optional = true}) + add_includedirs("include") + add_includedirs("include/torch/csrc/api/include") + + -- prevent the link to the libraries found automatically + add_links("") + + on_load("windows|x64", "macosx", "linux", function (package) + + -- ensure that git core.longpaths is enabled + os.vrun("git config --global core.longpaths true") + + if package:config("ninja") then + package:add("deps", "ninja") + end + + if not package:is_plat("macosx") then + if not find_package("mkl") then + package:add("deps", "openblas") + end + end + end) + + on_install("windows|x64", "macosx", "linux", function (package) + import("package.tools.cmake") + + -- tackle link flags + local has_cuda = package:dep("cuda"):exists() and package:dep("nvtx"):exists() + local libnames = {"torch", "torch_cpu"} + if has_cuda then + table.insert(libnames, "torch_cuda") + end + table.insert(libnames, "c10") + if has_cuda then + table.insert(libnames, "c10_cuda") + end + local suffix = "" + if not package:is_plat("windows") and package:config("shared") then + package:add("ldflags", "-Wl,-rpath," .. package:installdir("lib")) + if package:is_plat("linux") then + suffix = ".so" + elseif package:is_plat("macosx") then + suffix = ".dylib" + end + for _, lib in ipairs(libnames) do + package:add("ldflags", (package:is_plat("linux") and "-Wl,--no-as-needed," or "") .. package:installdir("lib", "lib") .. lib .. suffix) + end + else + for _, lib in ipairs(libnames) do + package:add("links", lib) + end + end + + -- workaround before 2.5.3, will be removed in future + if xmake.version():le("2.5.3") then + os.vrun("git submodule sync") + os.vrun("git submodule update --init --recursive") + end + + -- some patches to the third-party cmake files + io.replace("cmake/Modules/FindMKL.cmake", "MSVC AND NOT CMAKE_CXX_COMPILER_ID STREQUAL \"Intel\"", "FALSE", {plain = true}) + io.replace("third_party/fbgemm/CMakeLists.txt", "PRIVATE FBGEMM_STATIC", "PUBLIC FBGEMM_STATIC", {plain = true}) + io.replace("third_party/protobuf/cmake/install.cmake", "install%(DIRECTORY.-%)", "") + io.replace("third_party/ideep/mkl-dnn/src/CMakeLists.txt", "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}", "${CMAKE_INSTALL_LIBDIR}", {plain = true}) + if package:is_plat("windows") then + io.replace("cmake/Modules/FindOpenBLAS.cmake", "NAMES openblas PATHS", "NAMES libopenblas PATHS", {plain = true}) + if package:config("vs_runtime"):startswith("MD") then + io.replace("third_party/fbgemm/CMakeLists.txt", "MT", "MD", {plain = true}) + end + end + + -- prepare python + os.vrun("python -m pip install typing_extensions pyyaml") + local configs = {"-DUSE_MPI=OFF"} + if package:config("python") then + table.insert(configs, "-DBUILD_PYTHON=ON") + os.vrun("python -m pip install numpy") + else + table.insert(configs, "-DBUILD_PYTHON=OFF") + end + + -- prepare for installation + local extracfg = {} + if package:config("ninja") then + extracfg.cmake_generator = "Ninja" + end + local envs = cmake.buildenvs(package, extracfg) + if not package:is_plat("macosx") then + if package:dep("mkl"):exists() then + table.insert(configs, "-DBLAS=MKL") + local mkl = package:dep("mkl"):fetch() + table.insert(configs, "-DINTEL_MKL_DIR=" .. path.directory(mkl.sysincludedirs[1])) + else + table.insert(configs, "-DBLAS=OpenBLAS") + envs.OpenBLAS_HOME = package:dep("openblas"):installdir() + end + end + envs.libuv_ROOT = package:dep("libuv"):installdir() + table.insert(configs, "-DCMAKE_BUILD_TYPE=" .. (package:debug() and "Debug" or "Release")) + table.insert(configs, "-DBUILD_SHARED_LIBS=" .. (package:config("shared") and "ON" or "OFF")) + table.insert(configs, "-DCAFFE2_USE_MSVC_STATIC_RUNTIME=" .. (package:config("vs_runtime"):startswith("MT") and "ON" or "OFF")) + extracfg.envs = envs + cmake.install(package, configs, extracfg) + end) + + on_test(function (package) + assert(package:check_cxxsnippets({test = [[ + void test() { + auto a = torch::ones(3); + auto b = torch::tensor({1, 2, 3}); + auto c = torch::dot(a, b); + } + ]]}, {configs = {languages = "c++14"}, includes = "torch/torch.h"})) + end) diff --git a/packages/m/mkl/xmake.lua b/packages/m/mkl/xmake.lua new file mode 100644 index 00000000000..dd65cff3123 --- /dev/null +++ b/packages/m/mkl/xmake.lua @@ -0,0 +1,10 @@ +package("mkl") + + set_homepage("https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html") + set_description("Intel® oneAPI Math Kernel Library") + + on_fetch(function (package, opt) + if opt.system then + return find_package("mkl") + end + end) diff --git a/packages/n/nvtx/xmake.lua b/packages/n/nvtx/xmake.lua new file mode 100644 index 00000000000..8349955af13 --- /dev/null +++ b/packages/n/nvtx/xmake.lua @@ -0,0 +1,11 @@ +package("nvtx") + + set_homepage("https://github.com/NVIDIA/NVTX/") + set_description("The NVIDIA® Tools Extension SDK (NVTX) is a C-based Application Programming Interface (API) for annotating events, code ranges, and resources in your applications.") + + add_deps("cuda") + on_fetch(function (package, opt) + if opt.system then + return find_package("nvtx") + end + end) From 0b2d611a05adfe13c67d2cd27718a013720b9245 Mon Sep 17 00:00:00 2001 From: xq114 <1140735506@qq.com> Date: Wed, 31 Mar 2021 09:38:12 +0800 Subject: [PATCH 2/5] update to 1.8.1 --- packages/l/libtorch/xmake.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/l/libtorch/xmake.lua b/packages/l/libtorch/xmake.lua index 440bc254134..ba454e7aa40 100644 --- a/packages/l/libtorch/xmake.lua +++ b/packages/l/libtorch/xmake.lua @@ -6,6 +6,7 @@ package("libtorch") add_urls("https://github.com/pytorch/pytorch.git") add_versions("v1.8.0", "37c1f4a7fef115d719104e871d0cf39434aa9d56") + add_versions("v1.8.1", "56b43f4fec1f76953f15a627694d4bba34588969") add_configs("python", {description = "Build python interface.", default = false, type = "boolean"}) add_configs("ninja", {description = "Use ninja as build tool.", default = false, type = "boolean"}) From 80d729ddb6555ab0421e6683c5ea231cbd4e65c5 Mon Sep 17 00:00:00 2001 From: xq114 <1140735506@qq.com> Date: Wed, 31 Mar 2021 12:00:12 +0800 Subject: [PATCH 3/5] remove unused package stubs --- packages/m/mkl/xmake.lua | 10 ---------- packages/n/nvtx/xmake.lua | 11 ----------- 2 files changed, 21 deletions(-) delete mode 100644 packages/m/mkl/xmake.lua delete mode 100644 packages/n/nvtx/xmake.lua diff --git a/packages/m/mkl/xmake.lua b/packages/m/mkl/xmake.lua deleted file mode 100644 index dd65cff3123..00000000000 --- a/packages/m/mkl/xmake.lua +++ /dev/null @@ -1,10 +0,0 @@ -package("mkl") - - set_homepage("https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html") - set_description("Intel® oneAPI Math Kernel Library") - - on_fetch(function (package, opt) - if opt.system then - return find_package("mkl") - end - end) diff --git a/packages/n/nvtx/xmake.lua b/packages/n/nvtx/xmake.lua deleted file mode 100644 index 8349955af13..00000000000 --- a/packages/n/nvtx/xmake.lua +++ /dev/null @@ -1,11 +0,0 @@ -package("nvtx") - - set_homepage("https://github.com/NVIDIA/NVTX/") - set_description("The NVIDIA® Tools Extension SDK (NVTX) is a C-based Application Programming Interface (API) for annotating events, code ranges, and resources in your applications.") - - add_deps("cuda") - on_fetch(function (package, opt) - if opt.system then - return find_package("nvtx") - end - end) From 20c5c80a0e889bd7c3c6eda9bc17295450da4466 Mon Sep 17 00:00:00 2001 From: xq114 <1140735506@qq.com> Date: Wed, 31 Mar 2021 22:31:28 +0800 Subject: [PATCH 4/5] update for system package --- packages/c/cuda/xmake.lua | 4 +++- packages/l/libtorch/xmake.lua | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/c/cuda/xmake.lua b/packages/c/cuda/xmake.lua index b522e4cb5cf..181ffaf8759 100644 --- a/packages/c/cuda/xmake.lua +++ b/packages/c/cuda/xmake.lua @@ -18,7 +18,9 @@ package("cuda") table.insert(utils, package:config("shared") and "cudart" or "cudart_static") for _, util in ipairs(utils) do - if not find_library(util, cuda.linkdirs) then return end + if not find_library(util, cuda.linkdirs) then + return + end table.insert(result.links, util) end return result diff --git a/packages/l/libtorch/xmake.lua b/packages/l/libtorch/xmake.lua index ba454e7aa40..c482c0db307 100644 --- a/packages/l/libtorch/xmake.lua +++ b/packages/l/libtorch/xmake.lua @@ -15,7 +15,7 @@ package("libtorch") add_deps("python 3.x", {kind = "binary", system = false}) add_deps("libuv") add_deps("cuda", {optional = true, configs = {utils = {"nvrtc", "cudnn", "cufft", "curand", "cublas", "cudart_static"}}}) - add_deps("nvtx", "mkl", {optional = true}) + add_deps("nvtx", "mkl", {optional = true, system = true}) add_includedirs("include") add_includedirs("include/torch/csrc/api/include") From 1c43863135d80d29f8b006fe8627018fb6651723 Mon Sep 17 00:00:00 2001 From: xq114 <1140735506@qq.com> Date: Thu, 1 Apr 2021 16:25:40 +0800 Subject: [PATCH 5/5] remove git.core.longpaths setting --- packages/l/libtorch/xmake.lua | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/l/libtorch/xmake.lua b/packages/l/libtorch/xmake.lua index c482c0db307..6b37b2fca00 100644 --- a/packages/l/libtorch/xmake.lua +++ b/packages/l/libtorch/xmake.lua @@ -23,10 +23,6 @@ package("libtorch") add_links("") on_load("windows|x64", "macosx", "linux", function (package) - - -- ensure that git core.longpaths is enabled - os.vrun("git config --global core.longpaths true") - if package:config("ninja") then package:add("deps", "ninja") end