Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add libtorch #348

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions packages/c/cuda/xmake.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package("cuda")

set_homepage("https://developer.nvidia.com/cuda-zone/")
set_description("CUDA® is a parallel computing platform and programming model developed by NVIDIA for general computing on graphical processing units (GPUs).")

add_configs("utils", {description = "enabled cuda utilities.", default = {}, type = "table"})

on_fetch(function (package, opt)
if opt.system then
import("detect.sdks.find_cuda")
import("lib.detect.find_library")

local cuda = find_cuda()
if cuda then
package:addenv("PATH", cuda.bindir)
local result = {includedirs = cuda.includedirs, linkdirs = cuda.linkdirs, links = {}}
local utils = package:config("utils")
table.insert(utils, package:config("shared") and "cudart" or "cudart_static")

for _, util in ipairs(utils) do
if not find_library(util, cuda.linkdirs) then return end
table.insert(result.links, util)
xq114 marked this conversation as resolved.
Show resolved Hide resolved
end
return result
end
end
end)
131 changes: 131 additions & 0 deletions packages/l/libtorch/xmake.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
package("libtorch")

set_homepage("https://pytorch.org/")
set_description("An open source machine learning framework that accelerates the path from research prototyping to production deployment.")
set_license("BSD-3-Clause")

add_urls("https://github.com/pytorch/pytorch.git")
add_versions("v1.8.0", "37c1f4a7fef115d719104e871d0cf39434aa9d56")
add_versions("v1.8.1", "56b43f4fec1f76953f15a627694d4bba34588969")

add_configs("python", {description = "Build python interface.", default = false, type = "boolean"})
add_configs("ninja", {description = "Use ninja as build tool.", default = false, type = "boolean"})

add_deps("cmake")
add_deps("python 3.x", {kind = "binary", system = false})
add_deps("libuv")
add_deps("cuda", {optional = true, configs = {utils = {"nvrtc", "cudnn", "cufft", "curand", "cublas", "cudart_static"}}})
add_deps("nvtx", "mkl", {optional = true})
xq114 marked this conversation as resolved.
Show resolved Hide resolved
add_includedirs("include")
add_includedirs("include/torch/csrc/api/include")

-- prevent the link to the libraries found automatically
add_links("")

on_load("windows|x64", "macosx", "linux", function (package)

-- ensure that git core.longpaths is enabled
os.vrun("git config --global core.longpaths true")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个会影响用户的全局配置,不要这么做。。另外也不要放到 on_load 里面做。。

回头我看看在 xmake 内部对 local git repo 开启 git config core.longpaths true 试试,全局影响 不好。

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个删了吧,我 dev 上改进过了,git clone 的时候,会去设置 longpaths

git clone -c core.longpaths=true

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里光在clone的时候设置恐怕不行,报错是在git submodule update --recursive的时候发生的,主要是submodule嵌套才导致路径过长

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里光在clone的时候设置恐怕不行,报错是在git submodule update --recursive的时候发生的,主要是submodule嵌套才导致路径过长

只有 git clone 才支持 core.longpaths=true 参数设置。。

不过我看到 git clone 传递 longpaths 后,clone 下来的repo 的 本地 .git/config 配置里面已经默认生效 longpaths了。。

也就是说 后续的 一切操作 git pull/git submodule 按理应该默认都是 生效 longpaths 的。。

[core]
	longpaths = true

你可以 clone 一个确认下,我觉得现在应该就已经可以了,只需要 clone 时候 设置 longpaths,就能对这个 repo 永久生效。。

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

发现不行,并不生效

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

另一个思路是安装前获取longpath的设置,装完给他改回去

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

先放着吧,我晚上再研究下。。按理本地设置肯定能生效才对

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

另一个思路是安装前获取longpath的设置,装完给他改回去

只能试试这个了,今天尝试了下 git submodule foreach --recurisve git config core.langpaths true 去对每个 submodule 更新本地 longpaths 设置,也无效,蛋疼~

我后两天会提供个 set_policy("platform.longpaths", true) 的包接口设置,专门用来解决 longpaths 问题。。


if package:config("ninja") then
package:add("deps", "ninja")
end

if not package:is_plat("macosx") then
if not find_package("mkl") then
package:add("deps", "openblas")
end
end
end)

on_install("windows|x64", "macosx", "linux", function (package)
import("package.tools.cmake")

-- tackle link flags
local has_cuda = package:dep("cuda"):exists() and package:dep("nvtx"):exists()
local libnames = {"torch", "torch_cpu"}
if has_cuda then
table.insert(libnames, "torch_cuda")
end
table.insert(libnames, "c10")
if has_cuda then
table.insert(libnames, "c10_cuda")
end
local suffix = ""
if not package:is_plat("windows") and package:config("shared") then
package:add("ldflags", "-Wl,-rpath," .. package:installdir("lib"))
if package:is_plat("linux") then
suffix = ".so"
elseif package:is_plat("macosx") then
suffix = ".dylib"
end
for _, lib in ipairs(libnames) do
package:add("ldflags", (package:is_plat("linux") and "-Wl,--no-as-needed," or "") .. package:installdir("lib", "lib") .. lib .. suffix)
end
else
for _, lib in ipairs(libnames) do
package:add("links", lib)
end
end

-- workaround before 2.5.3, will be removed in future
if xmake.version():le("2.5.3") then
os.vrun("git submodule sync")
os.vrun("git submodule update --init --recursive")
end

-- some patches to the third-party cmake files
io.replace("cmake/Modules/FindMKL.cmake", "MSVC AND NOT CMAKE_CXX_COMPILER_ID STREQUAL \"Intel\"", "FALSE", {plain = true})
io.replace("third_party/fbgemm/CMakeLists.txt", "PRIVATE FBGEMM_STATIC", "PUBLIC FBGEMM_STATIC", {plain = true})
io.replace("third_party/protobuf/cmake/install.cmake", "install%(DIRECTORY.-%)", "")
io.replace("third_party/ideep/mkl-dnn/src/CMakeLists.txt", "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}", "${CMAKE_INSTALL_LIBDIR}", {plain = true})
if package:is_plat("windows") then
io.replace("cmake/Modules/FindOpenBLAS.cmake", "NAMES openblas PATHS", "NAMES libopenblas PATHS", {plain = true})
if package:config("vs_runtime"):startswith("MD") then
io.replace("third_party/fbgemm/CMakeLists.txt", "MT", "MD", {plain = true})
end
end

-- prepare python
os.vrun("python -m pip install typing_extensions pyyaml")
local configs = {"-DUSE_MPI=OFF"}
if package:config("python") then
table.insert(configs, "-DBUILD_PYTHON=ON")
os.vrun("python -m pip install numpy")
else
table.insert(configs, "-DBUILD_PYTHON=OFF")
end

-- prepare for installation
local extracfg = {}
if package:config("ninja") then
extracfg.cmake_generator = "Ninja"
end
local envs = cmake.buildenvs(package, extracfg)
if not package:is_plat("macosx") then
if package:dep("mkl"):exists() then
table.insert(configs, "-DBLAS=MKL")
local mkl = package:dep("mkl"):fetch()
table.insert(configs, "-DINTEL_MKL_DIR=" .. path.directory(mkl.sysincludedirs[1]))
else
table.insert(configs, "-DBLAS=OpenBLAS")
envs.OpenBLAS_HOME = package:dep("openblas"):installdir()
xq114 marked this conversation as resolved.
Show resolved Hide resolved
end
end
envs.libuv_ROOT = package:dep("libuv"):installdir()
table.insert(configs, "-DCMAKE_BUILD_TYPE=" .. (package:debug() and "Debug" or "Release"))
table.insert(configs, "-DBUILD_SHARED_LIBS=" .. (package:config("shared") and "ON" or "OFF"))
table.insert(configs, "-DCAFFE2_USE_MSVC_STATIC_RUNTIME=" .. (package:config("vs_runtime"):startswith("MT") and "ON" or "OFF"))
extracfg.envs = envs
cmake.install(package, configs, extracfg)
end)

on_test(function (package)
assert(package:check_cxxsnippets({test = [[
void test() {
auto a = torch::ones(3);
auto b = torch::tensor({1, 2, 3});
auto c = torch::dot(a, b);
}
]]}, {configs = {languages = "c++14"}, includes = "torch/torch.h"}))
end)