From 820add31717ba6a0570ee1a076d3713e62409483 Mon Sep 17 00:00:00 2001 From: SpaceIm <30052553+SpaceIm@users.noreply.github.com> Date: Wed, 31 Mar 2021 12:49:03 +0200 Subject: [PATCH] add libtorch/1.8.1 --- recipes/libtorch/all/CMakeLists.txt | 60 ++ recipes/libtorch/all/conandata.yml | 20 + recipes/libtorch/all/conanfile.py | 683 ++++++++++++++++++ .../0001-fix-system-onnx-namespace.patch | 13 + .../0002-allow-pytorch-subproject.patch | 48 ++ ...disable-pybind11-for-no-build-python.patch | 23 + ...id-python-external-typing-extensions.patch | 24 + .../0005-fix-vulkan-code-generation.patch | 43 ++ .../all/patches/0006-fix-install-target.patch | 279 +++++++ ...nvendor-dependencies-and-honor-flags.patch | 392 ++++++++++ .../libtorch/all/test_package/CMakeLists.txt | 11 + .../libtorch/all/test_package/conanfile.py | 17 + .../all/test_package/test_package.cpp | 31 + recipes/libtorch/config.yml | 3 + 14 files changed, 1647 insertions(+) create mode 100644 recipes/libtorch/all/CMakeLists.txt create mode 100644 recipes/libtorch/all/conandata.yml create mode 100644 recipes/libtorch/all/conanfile.py create mode 100644 recipes/libtorch/all/patches/0001-fix-system-onnx-namespace.patch create mode 100644 recipes/libtorch/all/patches/0002-allow-pytorch-subproject.patch create mode 100644 recipes/libtorch/all/patches/0003-disable-pybind11-for-no-build-python.patch create mode 100644 recipes/libtorch/all/patches/0004-avoid-python-external-typing-extensions.patch create mode 100644 recipes/libtorch/all/patches/0005-fix-vulkan-code-generation.patch create mode 100644 recipes/libtorch/all/patches/0006-fix-install-target.patch create mode 100644 recipes/libtorch/all/patches/0007-unvendor-dependencies-and-honor-flags.patch create mode 100644 recipes/libtorch/all/test_package/CMakeLists.txt create mode 100644 recipes/libtorch/all/test_package/conanfile.py create mode 100644 recipes/libtorch/all/test_package/test_package.cpp create mode 100644 recipes/libtorch/config.yml diff --git a/recipes/libtorch/all/CMakeLists.txt b/recipes/libtorch/all/CMakeLists.txt new file mode 100644 index 00000000000000..54e5616b576050 --- /dev/null +++ b/recipes/libtorch/all/CMakeLists.txt @@ -0,0 +1,60 @@ +cmake_minimum_required(VERSION 3.12) +project(cmake_wrapper) + +include(conanbuildinfo.cmake) +conan_basic_setup(TARGETS) + +#----------------------------- +# Workaround for github action +# see https://github.com/actions/setup-python/issues/121#issuecomment-777748504 +if (POLICY CMP0094) + cmake_policy(SET CMP0094 NEW) +endif () +if (NOT DEFINED Python_FIND_REGISTRY) + set(Python_FIND_REGISTRY "LAST") +endif () +if (NOT DEFINED Python_FIND_FRAMEWORK) + set(Python_FIND_FRAMEWORK "LAST") +endif () +#----------------------------- + +find_package(Python 3.8.0 REQUIRED) +set(PYTHON_EXECUTABLE ${Python_EXECUTABLE}) + +find_package(Eigen3 REQUIRED) +set(EIGEN3_FOUND 1) +set(EIGEN3_INCLUDE_DIR ${Eigen3_INCLUDE_DIR}) + +if(CONAN_LIBTORCH_USE_PTHREADPOOL) + add_library(pthreadpool INTERFACE IMPORTED) + set_property(TARGET pthreadpool PROPERTY INTERFACE_LINK_LIBRARIES CONAN_PKG::pthreadpool) +endif() + +if(CONAN_LIBTORCH_USE_CPUINFO) + add_library(cpuinfo INTERFACE IMPORTED) + add_library(clog INTERFACE IMPORTED) + set_property(TARGET cpuinfo PROPERTY INTERFACE_LINK_LIBRARIES CONAN_PKG::cpuinfo) + set_property(TARGET clog PROPERTY INTERFACE_LINK_LIBRARIES CONAN_PKG::cpuinfo) +endif() + +if(CONAN_LIBTORCH_USE_FXDIV) + add_library(fxdiv INTERFACE IMPORTED) + set_property(TARGET fxdiv PROPERTY INTERFACE_LINK_LIBRARIES CONAN_PKG::fxdiv) +endif() + +if(CONAN_LIBTORCH_USE_PSIMD) + add_library(psimd INTERFACE IMPORTED) + set_property(TARGET psimd PROPERTY INTERFACE_LINK_LIBRARIES CONAN_PKG::psimd) +endif() + +if(CONAN_LIBTORCH_USE_FP16) + add_library(fp16 INTERFACE IMPORTED) + set_property(TARGET fp16 PROPERTY INTERFACE_LINK_LIBRARIES CONAN_PKG::fp16) +endif() + +if(CONAN_LIBTORCH_USE_SLEEF) + add_library(sleef INTERFACE IMPORTED) + set_property(TARGET sleef PROPERTY INTERFACE_LINK_LIBRARIES CONAN_PKG::sleef) +endif() + +add_subdirectory(source_subfolder) diff --git a/recipes/libtorch/all/conandata.yml b/recipes/libtorch/all/conandata.yml new file mode 100644 index 00000000000000..b71aa9689a1d1d --- /dev/null +++ b/recipes/libtorch/all/conandata.yml @@ -0,0 +1,20 @@ +sources: + "1.8.1": + url: "https://github.com/pytorch/pytorch/archive/v1.8.1.tar.gz" + sha256: "a13b379d7acd2470e643b4fd54a3bd4a68eea6032153aa3fd705ba34718a32dc" +patches: + "1.8.1": + - patch_file: "patches/0001-fix-system-onnx-namespace.patch" + base_path: "source_subfolder" + - patch_file: "patches/0002-allow-pytorch-subproject.patch" + base_path: "source_subfolder" + - patch_file: "patches/0003-disable-pybind11-for-no-build-python.patch" + base_path: "source_subfolder" + - patch_file: "patches/0004-avoid-python-external-typing-extensions.patch" + base_path: "source_subfolder" + - patch_file: "patches/0005-fix-vulkan-code-generation.patch" + base_path: "source_subfolder" + - patch_file: "patches/0006-fix-install-target.patch" + base_path: "source_subfolder" + - patch_file: "patches/0007-unvendor-dependencies-and-honor-flags.patch" + base_path: "source_subfolder" diff --git a/recipes/libtorch/all/conanfile.py b/recipes/libtorch/all/conanfile.py new file mode 100644 index 00000000000000..5103dee6fe0d47 --- /dev/null +++ b/recipes/libtorch/all/conanfile.py @@ -0,0 +1,683 @@ +from conans import ConanFile, CMake, tools +from conans.errors import ConanInvalidConfiguration +import glob +import os +import textwrap + +required_conan_version = ">=1.33.0" + + +class LibtorchConan(ConanFile): + name = "libtorch" + description = "Tensors and Dynamic neural networks with strong GPU acceleration." + license = "BSD-3-Clause" + topics = ("conan", "libtorch", "pytorch", "machine-learning", + "deep-learning", "neural-network", "gpu", "tensor") + homepage = "https://pytorch.org" + url = "https://github.com/conan-io/conan-center-index" + + settings = "os", "arch", "compiler", "build_type" + options = { + "shared": [True, False], + "fPIC": [True, False], + "blas": ["eigen", "atlas", "openblas", "mkl", "veclib", "flame", "generic"], # generic means "whatever blas lib found" + "aten_parallel_backend": ["native", "openmp", "tbb"], + "with_cuda": [True, False], + "with_cudnn": [True, False], + "with_nvrtc": [True, False], + "with_tensorrt": [True, False], + "with_kineto": [True, False], + "with_rocm": [True, False], + "with_nccl": [True, False], + "with_fbgemm": [True, False], + "fakelowp": [True, False], + "with_ffmpeg": [True, False], + "with_gflags": [True, False], + "with_glog": [True, False], + "with_leveldb": [True, False], + "with_lmdb": [True, False], + "with_metal": [True, False], + "with_nnapi": [True, False], + "with_nnpack": [True, False], + "with_numa": [True, False], + "observers": [True, False], + "with_opencl": [True, False], + "with_opencv": [True, False], + "profiling": [True, False], + "with_qnnpack": [True, False], + "with_redis": [True, False], + "with_rocksdb": [True, False], + "with_snpe": [True, False], + "with_vulkan": [True, False], + "vulkan_shaderc_runtime": [True, False], + "vulkan_relaxed_precision": [True, False], + "with_xnnpack": [True, False], + "with_zmq": [True, False], + "with_zstd": [True, False], + "with_mkldnn": [True, False], + "distributed": [True, False], + "with_mpi": [True, False], + "with_gloo": [True, False], + "with_tensorpipe": [True, False], + "utilities": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + "blas": "openblas", # TODO: should be mkl on non mobile os + "aten_parallel_backend": "native", + "with_cuda": False, + "with_cudnn": True, + "with_nvrtc": False, + "with_tensorrt": False, + "with_kineto": False, # TODO: should be True + "with_rocm": False, + "with_nccl": True, + "with_fbgemm": False, # TODO: should be True + "fakelowp": False, + "with_ffmpeg": False, + "with_gflags": False, + "with_glog": False, + "with_leveldb": False, + "with_lmdb": False, + "with_metal": True, + "with_nnapi": False, + "with_nnpack": False, # TODO: should be True + "with_qnnpack": True, + "with_xnnpack": True, + "with_numa": True, + "observers": False, + "with_opencl": False, + "with_opencv": False, + "profiling": False, + "with_redis": False, + "with_rocksdb": False, + "with_snpe": False, + "with_vulkan": False, + "vulkan_shaderc_runtime": False, + "vulkan_relaxed_precision": False, + "with_zmq": False, + "with_zstd": False, + "with_mkldnn": False, + "distributed": True, + "with_mpi": True, + "with_gloo": False, # TODO: should be True + "with_tensorpipe": True, + "utilities": False, + } + + short_paths = True + + exports_sources = ["CMakeLists.txt", "patches/**"] + generators = "cmake", "cmake_find_package", "cmake_find_package_multi" + _cmake = None + + @property + def _source_subfolder(self): + return "source_subfolder" + + @property + def _build_subfolder(self): + return "build_subfolder" + + def config_options(self): + # Change default options for several OS + if self.settings.os in ["Android", "iOS"]: + self.options.blas = "eigen" + if self.settings.os not in ["Linux", "Windows"]: + self.options.distributed = False + + # Remove several options not supported for several OS + if self.settings.os == "Windows": + del self.options.fPIC + del self.options.with_nnpack + del self.options.with_qnnpack + del self.options.with_mpi + del self.options.with_tensorpipe + del self.options.with_kineto + if self.settings.os != "iOS": + del self.options.with_metal + if self.settings.os != "Android": + del self.options.with_nnapi + del self.options.with_snpe + if self.settings.os != "Linux": + del self.options.with_numa + + def configure(self): + if self.options.shared: + del self.options.fPIC + if not self.options.with_cuda: + del self.options.with_cudnn + del self.options.with_nvrtc + del self.options.with_tensorrt + del self.options.with_kineto + if not (self.options.with_cuda or self.options.with_rocm): + del self.options.with_nccl + if not self.options.with_vulkan: + del self.options.vulkan_shaderc_runtime + del self.options.vulkan_relaxed_precision + if not self.options.with_fbgemm: + del self.options.fakelowp + if not self.options.distributed: + del self.options.with_mpi + del self.options.with_gloo + del self.options.with_tensorpipe + + if self.settings.compiler.get_safe("cppstd"): + tools.check_min_cppstd(self, 14) + if self.options.with_cuda and self.options.with_rocm: + raise ConanInvalidConfiguration("libtorch doesn't yet support simultaneously building with CUDA and ROCm") + if self.options.with_ffmpeg and not self.options.with_opencv: + raise ConanInvalidConfiguration("libtorch video support with ffmpeg also requires opencv") + if self.options.blas == "veclib" and not tools.is_apple_os(self.settings.os): + raise ConanInvalidConfiguration("veclib only available on Apple family OS") + if self.settings.os == "Linux" and self.settings.compiler == "clang" and self.settings.compiler.libcxx == "libc++": + raise ConanInvalidConfiguration("clang with libc++ can't build libtorch") # TODO: try to fix that + + if self.options.distributed and self.settings.os not in ["Linux", "Windows"]: + self.output.warn("Distributed libtorch is not tested on {} and likely won't work".format(str(self.settings.os))) + + # numa static can't be linked into shared libs. + # Because Caffe2_detectron_ops* libs are always shared, we have to force + # libnuma shared even if libtorch:shared=False + if self.options.get_safe("with_numa"): + self.options["libnuma"].shared = True + + def requirements(self): + self.requires("cpuinfo/cci.20201217") + self.requires("eigen/3.3.9") + self.requires("fmt/7.1.3") + self.requires("foxi/cci.20210217") + self.requires("onnx/1.8.1") + self.requires("protobuf/3.15.5") + if self._depends_on_sleef: + self.requires("sleef/3.5.1") + if self.options.blas == "openblas": + self.requires("openblas/0.3.13") + elif self.options.blas in ["atlas", "mkl", "flame"]: + raise ConanInvalidConfiguration("{} recipe not yet available in CCI".format(self.options.blas)) + if self.options.aten_parallel_backend == "tbb": + self.requires("tbb/2020.3") + if self.options.with_cuda: + self.output.warn("cuda recipe not yet available in CCI, assuming that NVIDIA CUDA SDK is installed on your system") + if self.options.get_safe("with_cudnn"): + self.output.warn("cudnn recipe not yet available in CCI, assuming that NVIDIA CuDNN is installed on your system") + if self.options.get_safe("with_tensorrt"): + self.output.warn("tensorrt recipe not yet available in CCI, assuming that NVIDIA TensorRT SDK is installed on your system") + if self.options.get_safe("with_kineto"): + raise ConanInvalidConfiguration("kineto recipe not yet available in CCI") + if self.options.with_rocm: + raise ConanInvalidConfiguration("rocm recipe not yet available in CCI") + if self.options.with_fbgemm: + raise ConanInvalidConfiguration("fbgemm recipe not yet available in CCI") + self.requires("fbgemm/cci.20210309") + if self.options.with_ffmpeg: + raise ConanInvalidConfiguration("ffmpeg recipe not yet available in CCI") + if self.options.with_gflags: + self.requires("gflags/2.2.2") + if self.options.with_glog: + self.requires("glog/0.4.0") + if self.options.with_leveldb: + self.requires("leveldb/1.22") + if self.options.with_lmdb: + # should be part of OpenLDAP or packaged separately? + raise ConanInvalidConfiguration("lmdb recipe not yet available in CCI") + if self.options.get_safe("with_nnpack"): + raise ConanInvalidConfiguration("nnpack recipe not yet available in CCI") + if self.options.get_safe("with_qnnpack"): + self.requires("fp16/cci.20200514") + self.requires("fxdiv/cci.20200417") + self.requires("psimd/cci.20200517") + if self.options.with_xnnpack: + self.requires("xnnpack/cci.20210310") + if self.options.get_safe("with_nnpack") or self.options.get_safe("with_qnnpack") or self.options.with_xnnpack: + self.requires("pthreadpool/cci.20210218") + if self.options.get_safe("with_numa"): + self.requires("libnuma/2.0.14") + if self.options.with_opencl: + self.requires("opencl-headers/2020.06.16") + self.requires("opencl-icd-loader/2020.06.16") + if self.options.with_opencv: + self.requires("opencv/4.5.1") + if self.options.with_redis: + self.requires("hiredis/1.0.0") + if self.options.with_rocksdb: + self.requires("rocksdb/6.10.2") + if self.options.with_vulkan: + self.requires("vulkan-headers/1.2.170.0") + self.requires("vulkan-loader/1.2.170.0") + if self.options.get_safe("vulkan_shaderc_runtime"): + self.requires("shaderc/2019.0") + if self.options.with_zmq: + self.requires("zeromq/4.3.4") + if self.options.with_zstd: + self.requires("zstd/1.4.9") + if self.options.with_mkldnn: + raise ConanInvalidConfiguration("oneDNN (MKL-DNN) recipe not yet available in CCI") + if self.options.get_safe("with_mpi"): + self.requires("openmpi/4.1.0") + if self.options.get_safe("with_gloo"): + raise ConanInvalidConfiguration("gloo recipe not yet available in CCI") + if self.options.get_safe("with_tensorpipe"): + self.requires("tensorpipe/cci.20210316") + + @property + def _depends_on_sleef(self): + return self.settings.compiler != "Visual Studio" and self.settings.os not in ["Android", "iOS"] + + def validate(self): + if self.options.get_safe("with_numa") and not self.options["libnuma"].shared: + raise ConanInvalidConfiguration("libtorch requires libnuma shared. Set libnuma:shared=True, or disable " \ + "numa with libtorch:with_numa=False") + + def build_requirements(self): + if self.options.with_vulkan and not self.options.vulkan_shaderc_runtime: + self.build_requires("shaderc/2019.0") + # FIXME: libtorch 1.8.0 requires: + # - python 3.6.2+ with pyyaml, dataclasses and typing_extensions libs + # or + # - python 3.7+ with pyyaml and typing_extensions libs + # or + # - python 3.8+ with pyyaml lib + # self.build_requires("cpython/3.9.1") + + def source(self): + tools.get(**self.conan_data["sources"][self.version]) + os.rename("pytorch-" + self.version, self._source_subfolder) + + def _configure_cmake(self): + if self._cmake: + return self._cmake + self._cmake = CMake(self) + self._cmake.definitions["ATEN_NO_TEST"] = True + self._cmake.definitions["BUILD_BINARY"] = self.options.utilities + self._cmake.definitions["BUILD_DOCS"] = False + self._cmake.definitions["BUILD_CUSTOM_PROTOBUF"] = False + self._cmake.definitions["BUILD_PYTHON"] = False + self._cmake.definitions["BUILD_CAFFE2"] = True + self._cmake.definitions["BUILD_CAFFE2_OPS"] = True + self._cmake.definitions["BUILD_CAFFE2_MOBILE"] = False + self._cmake.definitions["CAFFE2_LINK_LOCAL_PROTOBUF"] = False + self._cmake.definitions["CAFFE2_USE_MSVC_STATIC_RUNTIME"] = self.settings.compiler.get_safe("runtime") in ["MT", "MTd", "static"] + self._cmake.definitions["BUILD_TEST"] = False + self._cmake.definitions["BUILD_STATIC_RUNTIME_BENCHMARK"] = False + self._cmake.definitions["BUILD_MOBILE_BENCHMARKS"] = False + self._cmake.definitions["BUILD_MOBILE_TEST"] = False + self._cmake.definitions["BUILD_JNI"] = False + self._cmake.definitions["BUILD_MOBILE_AUTOGRAD"] = False + self._cmake.definitions["INSTALL_TEST"] = False + self._cmake.definitions["USE_CPP_CODE_COVERAGE"] = False + self._cmake.definitions["COLORIZE_OUTPUT"] = True + self._cmake.definitions["USE_ASAN"] = False + self._cmake.definitions["USE_TSAN"] = False + self._cmake.definitions["USE_CUDA"] = self.options.with_cuda + self._cmake.definitions["USE_ROCM"] = self.options.with_rocm + self._cmake.definitions["CAFFE2_STATIC_LINK_CUDA"] = False + self._cmake.definitions["USE_CUDNN"] = self.options.get_safe("with_cudnn", False) + self._cmake.definitions["USE_STATIC_CUDNN"] = False + self._cmake.definitions["USE_FBGEMM"] = self.options.with_fbgemm + self._cmake.definitions["USE_KINETO"] = self.options.get_safe("with_kineto", False) + self._cmake.definitions["USE_FAKELOWP"] = self.options.get_safe("fakelowp", False) + self._cmake.definitions["USE_FFMPEG"] = self.options.with_ffmpeg + self._cmake.definitions["USE_GFLAGS"] = self.options.with_gflags + self._cmake.definitions["USE_GLOG"] = self.options.with_glog + self._cmake.definitions["USE_LEVELDB"] = self.options.with_leveldb + self._cmake.definitions["USE_LITE_PROTO"] = False + self._cmake.definitions["USE_LMDB"] = self.options.with_lmdb + self._cmake.definitions["USE_METAL"] = self.options.get_safe("with_metal", False) + self._cmake.definitions["USE_NATIVE_ARCH"] = False + self._cmake.definitions["USE_NCCL"] = self.options.get_safe("with_nccl", False) + self._cmake.definitions["USE_STATIC_NCCL"] = False + self._cmake.definitions["USE_SYSTEM_NCCL"] = False # technically we could create a recipe for nccl with 0 packages (because it requires CUDA at build time) + self._cmake.definitions["USE_NNAPI"] = self.options.get_safe("with_nnapi", False) + self._cmake.definitions["USE_NNPACK"] = self.options.get_safe("with_nnpack", False) + self._cmake.definitions["USE_NUMA"] = self.options.get_safe("with_numa", False) + self._cmake.definitions["USE_NVRTC"] = self.options.get_safe("with_nvrtc", False) + self._cmake.definitions["USE_NUMPY"] = False + self._cmake.definitions["USE_OBSERVERS"] = self.options.observers + self._cmake.definitions["USE_OPENCL"] = self.options.with_opencl + self._cmake.definitions["USE_OPENCV"] = self.options.with_opencv + self._cmake.definitions["USE_OPENMP"] = self.options.aten_parallel_backend == "openmp" + self._cmake.definitions["USE_PROF"] = self.options.profiling + self._cmake.definitions["USE_QNNPACK"] = False # QNNPACK is now integrated into libtorch and official repo + self._cmake.definitions["USE_PYTORCH_QNNPACK"] = self.options.get_safe("with_qnnpack", False) # is archived, so prefer to use vendored QNNPACK + self._cmake.definitions["USE_REDIS"] = self.options.with_redis + self._cmake.definitions["USE_ROCKSDB"] = self.options.with_rocksdb + self._cmake.definitions["USE_SNPE"] = self.options.get_safe("with_snpe", False) + self._cmake.definitions["USE_SYSTEM_EIGEN_INSTALL"] = True + self._cmake.definitions["USE_TENSORRT"] = self.options.get_safe("with_tensorrt", False) + self._cmake.definitions["USE_VULKAN"] = self.options.with_vulkan + self._cmake.definitions["USE_VULKAN_WRAPPER"] = False + self._cmake.definitions["USE_VULKAN_SHADERC_RUNTIME"] = self.options.get_safe("vulkan_shaderc_runtime", False) + self._cmake.definitions["USE_VULKAN_RELAXED_PRECISION"] = self.options.get_safe("vulkan_relaxed_precision", False) + self._cmake.definitions["USE_XNNPACK"] = self.options.with_xnnpack + self._cmake.definitions["USE_ZMQ"] = self.options.with_zmq + self._cmake.definitions["USE_ZSTD"] = self.options.with_zstd + self._cmake.definitions["USE_MKLDNN"] = self.options.with_mkldnn + self._cmake.definitions["USE_MKLDNN_CBLAS"] = False # This option has no logic and is useless in libtorch actually + self._cmake.definitions["USE_DISTRIBUTED"] = self.options.distributed + self._cmake.definitions["USE_MPI"] = self.options.get_safe("with_mpi", False) + self._cmake.definitions["USE_GLOO"] = self.options.get_safe("with_gloo", False) + self._cmake.definitions["USE_TENSORPIPE"] = self.options.get_safe("with_tensorpipe", False) + self._cmake.definitions["USE_TBB"] = self.options.aten_parallel_backend == "tbb" + self._cmake.definitions["ONNX_ML"] = True + self._cmake.definitions["HAVE_SOVERSION"] = True + self._cmake.definitions["USE_SYSTEM_LIBS"] = True + + self._cmake.definitions["USE_LAPACK"] = False # TODO: add an option + + self._cmake.definitions["BUILDING_WITH_TORCH_LIBS"] = True + self._cmake.definitions["BLAS"] = self._blas_cmake_option_value + + self._cmake.definitions["MSVC_Z7_OVERRIDE"] = False + + # Custom variables for our CMake wrapper + self._cmake.definitions["CONAN_LIBTORCH_USE_PTHREADPOOL"] = self._use_nnpack_family + self._cmake.definitions["CONAN_LIBTORCH_USE_CPUINFO"] = True + self._cmake.definitions["CONAN_LIBTORCH_USE_FXDIV"] = self.options.with_xnnpack + self._cmake.definitions["CONAN_LIBTORCH_USE_PSIMD"] = self.options.with_xnnpack + self._cmake.definitions["CONAN_LIBTORCH_USE_FP16"] = self.options.with_xnnpack + self._cmake.definitions["CONAN_LIBTORCH_USE_SLEEF"] = self._depends_on_sleef + + self._cmake.configure(build_folder=self._build_subfolder) + return self._cmake + + @property + def _blas_cmake_option_value(self): + return { + "eigen": "Eigen", + "atlas": "ATLAS", + "openblas": "OpenBLAS", + "mkl": "MKL", + "veclib": "vecLib", + "flame": "FLAME", + "generic": "Generic" + }[str(self.options.blas)] + + @property + def _use_nnpack_family(self): + return self.options.get_safe("with_nnpack") or self.options.get_safe("with_qnnpack") or self.options.with_xnnpack + + def build(self): + if self.options.get_safe("with_snpe"): + self.output.warn("with_snpe is enabled. Pay attention that you should have properly set SNPE_LOCATION and SNPE_HEADERS CMake variables") + for patch in self.conan_data.get("patches", {}).get(self.version, []): + tools.patch(**patch) + # conflict with macros.h generated at build time + os.remove(os.path.join(self.build_folder, self._source_subfolder, "caffe2", "core", "macros.h")) + cmake = self._configure_cmake() + cmake.build() + + def package(self): + self.copy("LICENSE", dst="licenses", src=self._source_subfolder) + cmake = self._configure_cmake() + cmake.install() # FIXME: something is wrong with includes layout + # TODO: Keep share/Aten/Declarations.yml? + tools.rmdir(os.path.join(self.package_folder, "share")) + tools.remove_files_by_mask(os.path.join(self.package_folder, "bin"), "*.pdb") + self._create_cmake_module_variables( + os.path.join(self.package_folder, self._module_subfolder, self._module_file) + ) + + @staticmethod + def _create_cmake_module_variables(module_file): + content = textwrap.dedent("""\ + if(DEFINED Torch_FOUND) + set(TORCH_FOUND ${Torch_FOUND}) + endif() + if(NOT DEFINED TORCH_INCLUDE_DIRS) + get_target_property(TORCH_INCLUDE_DIRS Torch::Torch INTERFACE_INCLUDE_DIRECTORIES) + endif() + if(NOT DEFINED TORCH_LIBRARIES) + set(TORCH_LIBRARIES "Torch::Torch") + endif() + """) + tools.save(module_file, content) + + @property + def _module_subfolder(self): + return os.path.join("lib", "cmake") + + @property + def _module_file(self): + return "conan-official-{}-variables.cmake".format(self.name) + + def package_info(self): + self.cpp_info.names["cmake_find_package"] = "Torch" + self.cpp_info.names["cmake_find_package_multi"] = "Torch" + + def _lib_exists(name): + return True if glob.glob(os.path.join(self.package_folder, "lib", "*{}.*".format(name))) else False + + def _add_whole_archive_lib(component, libname, shared=False): + if shared: + self.cpp_info.components[component].libs.append(libname) + else: + lib_folder = os.path.join(self.package_folder, "lib") + if self.settings.os == "Macos": + lib_fullpath = os.path.join(lib_folder, "lib{}.a".format(libname)) + whole_archive = "-Wl,-force_load,{}".format(lib_fullpath) + elif self.settings.compiler == "Visual Studio": + lib_fullpath = os.path.join(lib_folder, "{}".format(libname)) + whole_archive = "-WHOLEARCHIVE:{}".format(lib_fullpath) + else: + lib_fullpath = os.path.join(lib_folder, "lib{}.a".format(libname)) + whole_archive = "-Wl,--whole-archive,{},--no-whole-archive".format(lib_fullpath) + self.cpp_info.components[component].exelinkflags.append(whole_archive) + self.cpp_info.components[component].sharedlinkflags.append(whole_archive) + + def _sleef(): + return ["sleef::sleef"] if self._depends_on_sleef else [] + + def _openblas(): + return ["openblas::openblas"] if self.options.blas == "openblas" else [] + + def _tbb(): + return ["tbb::tbb"] if self.options.aten_parallel_backend == "tbb" else [] + + def _fbgemm(): + return ["fbgemm::fbgemm"] if self.options.with_fbgemm else [] + + def _ffmpeg(): + return ["ffmpeg::ffmpeg"] if self.options.with_ffmpeg else [] + + def _gflags(): + return ["gflags::gflags"] if self.options.with_gflags else [] + + def _glog(): + return ["glog::glog"] if self.options.with_glog else [] + + def _leveldb(): + return ["leveldb::leveldb"] if self.options.with_leveldb else [] + + def _nnpack(): + return ["nnpack::nnpack"] if self.options.get_safe("with_nnpack") else [] + + def _xnnpack(): + return ["xnnpack::xnnpack"] if self.options.with_xnnpack else [] + + def _pthreadpool(): + return ["pthreadpool::pthreadpool"] if self.options.get_safe("with_nnpack") or self.options.get_safe("with_qnnpack") or self.options.with_xnnpack else [] + + def _libnuma(): + return ["libnuma::libnuma"] if self.options.get_safe("with_numa") else [] + + def _opencl(): + return ["opencl-headers::opencl-headers", "opencl-icd-loader::opencl-icd-loader"] if self.options.with_opencl else [] + + def _opencv(): + return ["opencv::opencv"] if self.options.with_opencv else [] + + def _redis(): + return ["hiredis::hiredis"] if self.options.with_redis else [] + + def _vulkan(): + return ["vulkan-headers::vulkan-headers", "vulkan-loader::vulkan-loader"] if self.options.with_vulkan else [] + + def _shaderc(): + return ["shaderc::shaderc"] if self.options.get_safe("vulkan_shaderc_runtime") else [] + + def _zeromq(): + return ["zeromq::zeromq"] if self.options.with_zmq else [] + + def _zstd(): + return ["zstd::zstd"] if self.options.with_zstd else [] + + def _onednn(): + return ["onednn::onednn"] if self.options.with_mkldnn else [] + + def _openmpi(): + return ["openmpi::openmpi"] if self.options.get_safe("with_mpi") else [] + + def _gloo(): + return ["gloo::gloo"] if self.options.get_safe("with_gloo") else [] + + def _tensorpipe(): + return ["tensorpipe::tensorpipe"] if self.options.get_safe("with_tensorpipe") else [] + + # torch + _add_whole_archive_lib("_libtorch", "torch", shared=self.options.shared) + self.cpp_info.components["_libtorch"].requires.append("libtorch_cpu") + + # torch_cpu + _add_whole_archive_lib("libtorch_cpu", "torch_cpu", shared=self.options.shared) + self.cpp_info.components["libtorch_cpu"].requires.append("libtorch_c10") + + ## TODO: Eventually remove this workaround in the future + ## We put all these external dependencies and system libs of torch_cpu in an empty component instead, + ## due to "whole archive" trick. Indeed, conan doesn't honor libs order per component we expect in this case + ## (conan generators put exelinkflags/sharedlinkflags after system/external libs) + self.cpp_info.components["libtorch_cpu"].requires.append("libtorch_cpu_link_order_workaround") + self.cpp_info.components["libtorch_cpu_link_order_workaround"].requires.extend( + ["cpuinfo::cpuinfo", "eigen::eigen", "foxi::foxi"] + + _openblas() + _onednn() + _sleef() + _leveldb() + _openmpi() + + _gloo() + _redis() + _zstd() + _tensorpipe() + _opencv() + + _vulkan() + _shaderc() + _zeromq() + _ffmpeg() + ) + if self.settings.os == "Linux": + self.cpp_info.components["libtorch_cpu_link_order_workaround"].system_libs.extend(["dl", "m", "pthread", "rt"]) + if self.options.blas == "veclib": + self.cpp_info.components["libtorch_cpu_link_order_workaround"].frameworks.append("Accelerate") + + # c10 + self.cpp_info.components["libtorch_c10"].libs = ["c10"] + self.cpp_info.components["libtorch_c10"].requires.extend( + _gflags() + _glog() + _libnuma() + ) + if self.settings.os == "Android": + self.cpp_info.components["libtorch_c10"].system_libs.append("log") + + ##------------------ + ## FIXME: let's put all build modules, include dirs, external dependencies (except protobuf) and system/frameworks libs in c10 for the moment + self.cpp_info.components["libtorch_c10"].builddirs.append(self._module_subfolder) + self.cpp_info.components["libtorch_c10"].build_modules["cmake_find_package"] = [os.path.join(self._module_subfolder, self._module_file)] + self.cpp_info.components["libtorch_c10"].build_modules["cmake_find_package_multi"] = [os.path.join(self._module_subfolder, self._module_file)] + self.cpp_info.components["libtorch_c10"].includedirs.append(os.path.join("include", "torch", "csrc", "api", "include")) + self.cpp_info.components["libtorch_c10"].requires.extend(["fmt::fmt", "onnx::onnx"]) + self.cpp_info.components["libtorch_c10"].requires.extend( + _tbb() + _fbgemm() + _nnpack() + _xnnpack() + _pthreadpool() + + _opencl() + ) + ##------------------ + + if self.options.shared: + ## TODO: Eventually remove this workaround in the future + self.cpp_info.components["libtorch_cpu_link_order_workaround"].requires.append("protobuf::protobuf") + else: + # caffe2_protos + _add_whole_archive_lib("libtorch_caffe2_protos", "caffe2_protos") + self.cpp_info.components["libtorch_cpu"].requires.append("libtorch_caffe2_protos") + ## TODO: Eventually remove this workaround in the future + self.cpp_info.components["libtorch_caffe2_protos"].requires.append("libtorch_caffe2_protos_link_order_workaround") + self.cpp_info.components["libtorch_caffe2_protos_link_order_workaround"].requires.append("protobuf::protobuf") + + # Caffe2_perfkernels_avx + if _lib_exists("Caffe2_perfkernels_avx"): + _add_whole_archive_lib("libtorch_caffe2_perfkernels_avx", "Caffe2_perfkernels_avx", shared=self.options.shared) + self.cpp_info.components["libtorch_caffe2_perfkernels_avx"].requires.append("libtorch_c10") + self.cpp_info.components["libtorch_cpu"].requires.append("libtorch_caffe2_perfkernels_avx") + + # Caffe2_perfkernels_avx2 + if _lib_exists("Caffe2_perfkernels_avx2"): + _add_whole_archive_lib("libtorch_caffe2_perfkernels_avx2", "Caffe2_perfkernels_avx2", shared=self.options.shared) + self.cpp_info.components["libtorch_caffe2_perfkernels_avx2"].requires.append("libtorch_c10") + self.cpp_info.components["libtorch_cpu"].requires.append("libtorch_caffe2_perfkernels_avx2") + + # Caffe2_perfkernels_avx512 + if _lib_exists("Caffe2_perfkernels_avx512"): + _add_whole_archive_lib("libtorch_caffe2_perfkernels_avx512", "Caffe2_perfkernels_avx512", shared=self.options.shared) + self.cpp_info.components["libtorch_caffe2_perfkernels_avx512"].requires.append("libtorch_c10") + self.cpp_info.components["libtorch_cpu"].requires.append("libtorch_caffe2_perfkernels_avx512") + + # caffe2_observers + if self.options.observers: + _add_whole_archive_lib("libtorch_caffe2_observers", "caffe2_observers", shared=self.options.shared) + self.cpp_info.components["libtorch_caffe2_observers"].requires.append("_libtorch") + + # c10d + if self.options.distributed: + self.cpp_info.components["libtorch_c10d"].libs = ["c10d"] # always static + self.cpp_info.components["libtorch_c10d"].requires.extend(["_libtorch"] + _openmpi() + _gloo()) + + # process_group_agent & tensorpipe_agent + if self.options.get_safe("with_tensorpipe"): + self.cpp_info.components["libtorch_process_group_agent"].libs = ["process_group_agent"] + self.cpp_info.components["libtorch_process_group_agent"].requires.extend(["_libtorch", "libtorch_c10d"]) + self.cpp_info.components["libtorch_tensorpipe_agent"].libs = ["tensorpipe_agent"] + self.cpp_info.components["libtorch_tensorpipe_agent"].requires.extend(["_libtorch", "libtorch_c10d", "fmt::fmt"] + _tensorpipe()) + + # caffe2_nvrtc + if self.options.with_cuda or self.options.with_rocm: + self.cpp_info.components["libtorch_caffe2_nvrtc"].libs = ["caffe2_nvrtc"] + + if self.options.with_cuda: + # torch_cuda + _add_whole_archive_lib("libtorch_torch_cuda", "torch_cuda", shared=self.options.shared) + self.cpp_info.components["libtorch_torch_cuda"].requires.append("libtorch_c10_cuda") + self.cpp_info.components["_libtorch"].requires.append("libtorch_torch_cuda") + + # c10_cuda + self.cpp_info.components["libtorch_c10_cuda"].libs = ["c10_cuda"] + self.cpp_info.components["libtorch_c10_cuda"].requires.append("libtorch_c10") + + # caffe2_detectron_ops_gpu + if self.options.shared: + self.cpp_info.components["libtorch_caffe2_detectron_ops_gpu"].libs = ["caffe2_detectron_ops_gpu"] + self.cpp_info.components["libtorch_caffe2_detectron_ops_gpu"].requires.extend(["_libtorch", "libtorch_cpu", "libtorch_c10"]) + elif self.options.with_rocm: + # torch_hip + _add_whole_archive_lib("libtorch_torch_hip", "torch_hip", shared=self.options.shared) + self.cpp_info.components["libtorch_torch_hip"].requires.append("libtorch_c10_hip") + self.cpp_info.components["_libtorch"].requires.append("libtorch_torch_hip") + + # c10_hip + self.cpp_info.components["libtorch_c10_hip"].libs = ["c10_hip"] + self.cpp_info.components["libtorch_c10_hip"].requires.append("libtorch_c10") + + # caffe2_detectron_ops_hip + if self.options.shared: + self.cpp_info.components["libtorch_caffe2_detectron_ops_hip"].libs = ["caffe2_detectron_ops_hip"] + self.cpp_info.components["libtorch_caffe2_detectron_ops_hip"].requires.extend(["_libtorch", "libtorch_cpu", "libtorch_c10"]) + elif not self.settings.os == "iOS": + # caffe2_detectron_ops + if self.options.shared: + self.cpp_info.components["libtorch_caffe2_detectron_ops"].libs = ["caffe2_detectron_ops"] + self.cpp_info.components["libtorch_caffe2_detectron_ops"].requires.extend(["_libtorch", "libtorch_cpu", "libtorch_c10"]) + + # pytorch_qnnpack + if self.options.get_safe("with_qnnpack"): + self.cpp_info.components["libtorch_pytorch_qnnpack"].libs = ["pytorch_qnnpack"] + self.cpp_info.components["libtorch_pytorch_qnnpack"].requires.extend([ + "cpuinfo::cpuinfo", "fp16::fp16", "fxdiv::fxdiv", "psimd::psimd", "pthreadpool::pthreadpool" + ]) + self.cpp_info.components["libtorch_cpu"].requires.append("libtorch_pytorch_qnnpack") + + # caffe2_rocksdb + if self.options.with_rocksdb: + self.cpp_info.components["libtorch_caffe2_rocksdb"].libs = ["caffe2_rocksdb"] + self.cpp_info.components["libtorch_caffe2_rocksdb"].requires.extend(["_libtorch", "rocksdb::rocksdb"]) + + if self.options.utilities: + bin_path = os.path.join(self.package_folder, "bin") + self.output.info("Appending PATH environment variable: {}".format(bin_path)) + self.env_info.PATH.append(bin_path) diff --git a/recipes/libtorch/all/patches/0001-fix-system-onnx-namespace.patch b/recipes/libtorch/all/patches/0001-fix-system-onnx-namespace.patch new file mode 100644 index 00000000000000..40bba38efb5d97 --- /dev/null +++ b/recipes/libtorch/all/patches/0001-fix-system-onnx-namespace.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 24d3c9b3a4..35974f5bd1 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -336,7 +336,7 @@ cmake_dependent_option( + + if(NOT USE_SYSTEM_ONNX) + set(ONNX_NAMESPACE "onnx_torch" CACHE STRING "A namespace for ONNX; needed to build with other frameworks that share ONNX.") +-elseif() ++else() + set(ONNX_NAMESPACE "onnx" CACHE STRING "A namespace for ONNX; needed to build with other frameworks that share ONNX.") + endif() + set(SELECTED_OP_LIST "" CACHE STRING diff --git a/recipes/libtorch/all/patches/0002-allow-pytorch-subproject.patch b/recipes/libtorch/all/patches/0002-allow-pytorch-subproject.patch new file mode 100644 index 00000000000000..f07cb216e03138 --- /dev/null +++ b/recipes/libtorch/all/patches/0002-allow-pytorch-subproject.patch @@ -0,0 +1,48 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 35974f5bd1..ef677702ce 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -792,7 +792,7 @@ include_directories(BEFORE ${PROJECT_SOURCE_DIR}) + include_directories(BEFORE ${PROJECT_BINARY_DIR}) + + include_directories(BEFORE ${PROJECT_SOURCE_DIR}/aten/src/) +-include_directories(BEFORE ${PROJECT_BINARY_DIR}/aten/src/) ++include_directories(BEFORE ${CMAKE_BINARY_DIR}/aten/src/) + + # ---[ Main build + add_subdirectory(c10) +diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt +index 7bef849c08..e1da906b61 100644 +--- a/caffe2/CMakeLists.txt ++++ b/caffe2/CMakeLists.txt +@@ -60,7 +60,7 @@ if(INTERN_BUILD_ATEN_OPS) + "${PYTHON_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/contrib/aten/gen_op.py + --aten_root=${CMAKE_CURRENT_SOURCE_DIR}/../aten + --template_dir=${CMAKE_CURRENT_SOURCE_DIR}/contrib/aten +- --yaml_dir=${CMAKE_CURRENT_BINARY_DIR}/../aten/src/ATen ++ --yaml_dir=${CMAKE_BINARY_DIR}/aten/src/ATen + --install_dir=${CMAKE_CURRENT_BINARY_DIR}/contrib/aten + DEPENDS + ATEN_CPU_FILES_GEN_TARGET +@@ -581,7 +581,7 @@ if(NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE) + set(DELAY_LOAD_FLAGS "") + endif() + target_link_libraries(caffe2_nvrtc ${CUDA_NVRTC} ${CUDA_CUDA_LIB} ${CUDA_NVRTC_LIB} ${DELAY_LOAD_FLAGS}) +- target_include_directories(caffe2_nvrtc PRIVATE ${CUDA_INCLUDE_DIRS}) ++ target_include_directories(caffe2_nvrtc PRIVATE ${CUDA_INCLUDE_DIRS} ${CMAKE_BINARY_DIR}) + install(TARGETS caffe2_nvrtc DESTINATION "${TORCH_INSTALL_LIB_DIR}") + if(USE_NCCL AND BUILD_SPLIT_CUDA) + list(APPEND Caffe2_GPU_SRCS_CPP +diff --git a/cmake/Codegen.cmake b/cmake/Codegen.cmake +index a9d2e4f50e..a8bd3a2a94 100644 +--- a/cmake/Codegen.cmake ++++ b/cmake/Codegen.cmake +@@ -233,7 +233,7 @@ if(INTERN_BUILD_ATEN_OPS) + endif() + + function(append_filelist name outputvar) +- set(_rootdir "${${CMAKE_PROJECT_NAME}_SOURCE_DIR}/") ++ set(_rootdir "${Torch_SOURCE_DIR}/") + # configure_file adds its input to the list of CMAKE_RERUN dependencies + configure_file( + ${PROJECT_SOURCE_DIR}/tools/build_variables.bzl diff --git a/recipes/libtorch/all/patches/0003-disable-pybind11-for-no-build-python.patch b/recipes/libtorch/all/patches/0003-disable-pybind11-for-no-build-python.patch new file mode 100644 index 00000000000000..843df024dd1dae --- /dev/null +++ b/recipes/libtorch/all/patches/0003-disable-pybind11-for-no-build-python.patch @@ -0,0 +1,23 @@ +diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake +index 06464e799a..72b7d2809f 100644 +--- a/cmake/Dependencies.cmake ++++ b/cmake/Dependencies.cmake +@@ -992,6 +992,7 @@ if(BUILD_PYTHON) + endif() + + # ---[ pybind11 ++if(BUILD_PYTHON) + if(NOT pybind11_PREFER_third_party) + find_package(pybind11 CONFIG) + if(NOT pybind11_FOUND) +@@ -1013,6 +1014,10 @@ else() + endif() + message(STATUS "pybind11 include dirs: " "${pybind11_INCLUDE_DIRS}") + include_directories(SYSTEM ${pybind11_INCLUDE_DIRS}) ++else() ++ find_package(Python 3.8.0 REQUIRED) ++ set(PYTHON_EXECUTABLE ${Python_EXECUTABLE}) ++endif() + + # ---[ MPI + if(USE_MPI) diff --git a/recipes/libtorch/all/patches/0004-avoid-python-external-typing-extensions.patch b/recipes/libtorch/all/patches/0004-avoid-python-external-typing-extensions.patch new file mode 100644 index 00000000000000..69e3098b958108 --- /dev/null +++ b/recipes/libtorch/all/patches/0004-avoid-python-external-typing-extensions.patch @@ -0,0 +1,24 @@ +diff --git a/tools/codegen/dest/register_dispatch_key.py b/tools/codegen/dest/register_dispatch_key.py +index 6ab7f58843..87e762116b 100644 +--- a/tools/codegen/dest/register_dispatch_key.py ++++ b/tools/codegen/dest/register_dispatch_key.py +@@ -1,6 +1,6 @@ + from typing import List, Optional, Union + import itertools +-from typing_extensions import Literal ++from typing import Literal + from dataclasses import dataclass + + from tools.codegen.context import * +diff --git a/tools/codegen/gen.py b/tools/codegen/gen.py +index 4b8e59d2b2..ab4c17a461 100644 +--- a/tools/codegen/gen.py ++++ b/tools/codegen/gen.py +@@ -1,6 +1,6 @@ + import os + from typing import List, Dict, Optional, Tuple, Set, Callable, Any, Union, Sequence +-from typing_extensions import Literal ++from typing import Literal + import yaml + from collections import OrderedDict, defaultdict + import argparse diff --git a/recipes/libtorch/all/patches/0005-fix-vulkan-code-generation.patch b/recipes/libtorch/all/patches/0005-fix-vulkan-code-generation.patch new file mode 100644 index 00000000000000..d757c39ae69339 --- /dev/null +++ b/recipes/libtorch/all/patches/0005-fix-vulkan-code-generation.patch @@ -0,0 +1,43 @@ +diff --git a/cmake/VulkanCodegen.cmake b/cmake/VulkanCodegen.cmake +index 51e6b4bbf5..ac01d49ed6 100644 +--- a/cmake/VulkanCodegen.cmake ++++ b/cmake/VulkanCodegen.cmake +@@ -12,7 +12,11 @@ endif() + + if(USE_VULKAN_SHADERC_RUNTIME) + set(PYTHONPATH "$ENV{PYTHONPATH}") +- set(ENV{PYTHONPATH} "$ENV{PYTHONPATH}:${CMAKE_CURRENT_LIST_DIR}/..") ++ if(WIN32) ++ set(ENV{PYTHONPATH} "$ENV{PYTHONPATH};${CMAKE_CURRENT_LIST_DIR}/..") ++ else() ++ set(ENV{PYTHONPATH} "$ENV{PYTHONPATH}:${CMAKE_CURRENT_LIST_DIR}/..") ++ endif() + execute_process( + COMMAND + "${PYTHON_EXECUTABLE}" +@@ -41,11 +45,7 @@ if(NOT USE_VULKAN_SHADERC_RUNTIME) + + set(GLSLC_PATH "${ANDROID_NDK}/shader-tools/${ANDROID_NDK_HOST_SYSTEM_NAME}/glslc") + else() +- find_program( +- GLSLC_PATH glslc +- PATHS +- ENV VULKAN_SDK +- PATHS "$ENV{VULKAN_SDK}/${CMAKE_HOST_SYSTEM_PROCESSOR}/bin") ++ find_program(GLSLC_PATH glslc) + + if(NOT GLSLC_PATH) + message(FATAL_ERROR "USE_VULKAN glslc not found") +@@ -53,7 +53,11 @@ if(NOT USE_VULKAN_SHADERC_RUNTIME) + endif() + + set(PYTHONPATH "$ENV{PYTHONPATH}") +- set(ENV{PYTHONPATH} "$ENV{PYTHONPATH}:${CMAKE_CURRENT_LIST_DIR}/..") ++ if(WIN32) ++ set(ENV{PYTHONPATH} "$ENV{PYTHONPATH};${CMAKE_CURRENT_LIST_DIR}/..") ++ else() ++ set(ENV{PYTHONPATH} "$ENV{PYTHONPATH}:${CMAKE_CURRENT_LIST_DIR}/..") ++ endif() + execute_process( + COMMAND + "${PYTHON_EXECUTABLE}" diff --git a/recipes/libtorch/all/patches/0006-fix-install-target.patch b/recipes/libtorch/all/patches/0006-fix-install-target.patch new file mode 100644 index 00000000000000..029aaef3da519e --- /dev/null +++ b/recipes/libtorch/all/patches/0006-fix-install-target.patch @@ -0,0 +1,279 @@ +diff --git a/c10/CMakeLists.txt b/c10/CMakeLists.txt +index b175e5bdd6..a267af43d1 100644 +--- a/c10/CMakeLists.txt ++++ b/c10/CMakeLists.txt +@@ -106,7 +106,10 @@ endif() + # Note: for now, we will put all export path into one single Caffe2Targets group + # to deal with the cmake deployment need. Inside the Caffe2Targets set, the + # individual libraries like libc10.so and libcaffe2.so are still self-contained. +-install(TARGETS c10 EXPORT Caffe2Targets DESTINATION lib) ++install(TARGETS c10 EXPORT Caffe2Targets ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + DESTINATION include + FILES_MATCHING PATTERN "*.h") +@@ -114,5 +117,5 @@ install(FILES ${CMAKE_BINARY_DIR}/c10/macros/cmake_macros.h + DESTINATION include/c10/macros) + + if(MSVC AND C10_BUILD_SHARED_LIBS) +- install(FILES $ DESTINATION lib OPTIONAL) ++ install(FILES $ DESTINATION ${CMAKE_INSTALL_BINDIR} OPTIONAL) + endif() +diff --git a/c10/cuda/CMakeLists.txt b/c10/cuda/CMakeLists.txt +index 256fc54b08..d547f0040b 100644 +--- a/c10/cuda/CMakeLists.txt ++++ b/c10/cuda/CMakeLists.txt +@@ -64,7 +64,10 @@ add_subdirectory(test) + # Note: for now, we will put all export path into one single Caffe2Targets group + # to deal with the cmake deployment need. Inside the Caffe2Targets set, the + # individual libraries like libc10.so and libcaffe2.so are still self-contained. +-install(TARGETS c10_cuda EXPORT Caffe2Targets DESTINATION lib) ++install(TARGETS c10_cuda EXPORT Caffe2Targets ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + foreach(file ${C10_CUDA_HEADERS}) + get_filename_component( dir ${file} DIRECTORY ) + install( FILES ${file} DESTINATION include/c10/cuda/${dir} ) +@@ -73,5 +76,5 @@ install(FILES ${CMAKE_BINARY_DIR}/c10/cuda/impl/cuda_cmake_macros.h + DESTINATION include/c10/cuda/impl) + + if(MSVC AND C10_CUDA_BUILD_SHARED_LIBS) +- install(FILES $ DESTINATION lib OPTIONAL) ++ install(FILES $ DESTINATION ${CMAKE_INSTALL_BINDIR} OPTIONAL) + endif() +diff --git a/c10/hip/CMakeLists.txt b/c10/hip/CMakeLists.txt +index 6a0e0e41a1..424099c56e 100644 +--- a/c10/hip/CMakeLists.txt ++++ b/c10/hip/CMakeLists.txt +@@ -55,7 +55,10 @@ target_include_directories( + add_subdirectory(test) + + # ---[ Installation +-install(TARGETS c10_hip EXPORT Caffe2Targets DESTINATION lib) ++install(TARGETS c10_hip EXPORT Caffe2Targets ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + DESTINATION include + FILES_MATCHING PATTERN "*.h") +diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt +index e1da906b61..8eeb54e653 100644 +--- a/caffe2/CMakeLists.txt ++++ b/caffe2/CMakeLists.txt +@@ -350,6 +350,7 @@ if(NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE) + add_library(process_group_agent "${TORCH_SRC_DIR}/csrc/distributed/rpc/process_group_agent.cpp" "${TORCH_SRC_DIR}/csrc/distributed/rpc/process_group_agent.h") + target_link_libraries(process_group_agent PRIVATE torch c10d fmt::fmt-header-only) + add_dependencies(process_group_agent torch c10d) ++ install(TARGETS process_group_agent DESTINATION ${CMAKE_INSTALL_LIBDIR}) + + add_library(tensorpipe_agent + "${TORCH_SRC_DIR}/csrc/distributed/rpc/macros.h" +@@ -375,6 +376,7 @@ if(NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE) + target_compile_definitions(tensorpipe_agent PUBLIC USE_TENSORPIPE) + target_link_libraries(tensorpipe_agent PRIVATE tensorpipe) + add_dependencies(tensorpipe_agent tensorpipe) ++ install(TARGETS tensorpipe_agent DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() + endif() + endif() +@@ -582,7 +584,10 @@ if(NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE) + endif() + target_link_libraries(caffe2_nvrtc ${CUDA_NVRTC} ${CUDA_CUDA_LIB} ${CUDA_NVRTC_LIB} ${DELAY_LOAD_FLAGS}) + target_include_directories(caffe2_nvrtc PRIVATE ${CUDA_INCLUDE_DIRS} ${CMAKE_BINARY_DIR}) +- install(TARGETS caffe2_nvrtc DESTINATION "${TORCH_INSTALL_LIB_DIR}") ++ install(TARGETS caffe2_nvrtc ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + if(USE_NCCL AND BUILD_SPLIT_CUDA) + list(APPEND Caffe2_GPU_SRCS_CPP + ${TORCH_SRC_DIR}/csrc/cuda/nccl.cpp) +@@ -607,7 +612,10 @@ if(NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE) + add_library(caffe2_nvrtc SHARED ${ATen_NVRTC_STUB_SRCS}) + target_link_libraries(caffe2_nvrtc ${PYTORCH_HIP_HCC_LIBRARIES} ${ROCM_HIPRTC_LIB}) + target_compile_definitions(caffe2_nvrtc PRIVATE USE_ROCM __HIP_PLATFORM_HCC__) +- install(TARGETS caffe2_nvrtc DESTINATION "${TORCH_INSTALL_LIB_DIR}") ++ install(TARGETS caffe2_nvrtc ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() + + if(NOT NO_API) +@@ -1253,18 +1261,36 @@ endif() + + caffe2_interface_library(torch torch_library) + +-install(TARGETS torch_cpu torch_cpu_library EXPORT Caffe2Targets DESTINATION "${TORCH_INSTALL_LIB_DIR}") ++install(TARGETS torch_cpu torch_cpu_library EXPORT Caffe2Targets ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + + if(USE_CUDA) +- install(TARGETS torch_cuda torch_cuda_library EXPORT Caffe2Targets DESTINATION "${TORCH_INSTALL_LIB_DIR}") ++ install(TARGETS torch_cuda torch_cuda_library EXPORT Caffe2Targets ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + if(BUILD_SPLIT_CUDA) +- install(TARGETS torch_cuda_cu torch_cuda_cu_library EXPORT Caffe2Targets DESTINATION "${TORCH_INSTALL_LIB_DIR}") +- install(TARGETS torch_cuda_cpp torch_cuda_cpp_library EXPORT Caffe2Targets DESTINATION "${TORCH_INSTALL_LIB_DIR}") ++ install(TARGETS torch_cuda_cu torch_cuda_cu_library EXPORT Caffe2Targets ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) ++ install(TARGETS torch_cuda_cpp torch_cuda_cpp_library EXPORT Caffe2Targets ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() + elseif(USE_ROCM) +- install(TARGETS torch_hip torch_hip_library EXPORT Caffe2Targets DESTINATION "${TORCH_INSTALL_LIB_DIR}") ++ install(TARGETS torch_hip torch_hip_library EXPORT Caffe2Targets ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() +-install(TARGETS torch torch_library EXPORT Caffe2Targets DESTINATION "${TORCH_INSTALL_LIB_DIR}") ++install(TARGETS torch torch_library EXPORT Caffe2Targets ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + + target_link_libraries(torch PUBLIC torch_cpu_library) + +@@ -1280,14 +1306,14 @@ endif() + + # Install PDB files for MSVC builds + if(MSVC AND BUILD_SHARED_LIBS) +- install(FILES $ DESTINATION "${TORCH_INSTALL_LIB_DIR}" OPTIONAL) ++ install(FILES $ DESTINATION ${CMAKE_INSTALL_BINDIR} OPTIONAL) + if(BUILD_SPLIT_CUDA) +- install(FILES $ DESTINATION "${TORCH_INSTALL_LIB_DIR}" OPTIONAL) +- install(FILES $ DESTINATION "${TORCH_INSTALL_LIB_DIR}" OPTIONAL) ++ install(FILES $ DESTINATION ${CMAKE_INSTALL_BINDIR} OPTIONAL) ++ install(FILES $ DESTINATION ${CMAKE_INSTALL_BINDIR} OPTIONAL) + elseif(USE_CUDA) +- install(FILES $ DESTINATION "${TORCH_INSTALL_LIB_DIR}" OPTIONAL) ++ install(FILES $ DESTINATION ${CMAKE_INSTALL_BINDIR} OPTIONAL) + elseif(USE_ROCM) +- install(FILES $ DESTINATION "${TORCH_INSTALL_LIB_DIR}" OPTIONAL) ++ install(FILES $ DESTINATION ${CMAKE_INSTALL_BINDIR} OPTIONAL) + endif() + endif() + +diff --git a/caffe2/perfkernels/CMakeLists.txt b/caffe2/perfkernels/CMakeLists.txt +index 1fe8bca697..2fa168fd0e 100644 +--- a/caffe2/perfkernels/CMakeLists.txt ++++ b/caffe2/perfkernels/CMakeLists.txt +@@ -56,6 +56,11 @@ if(CAFFE2_COMPILER_SUPPORTS_AVX2_EXTENSIONS) + list(APPEND + Caffe2_DEPENDENCY_WHOLE_LINK_LIBS + "Caffe2_perfkernels_avx2_interface") ++ if(NOT BUILD_SHARED_LIBS) ++ install(TARGETS Caffe2_perfkernels_avx Caffe2_perfkernels_avx2 ++ EXPORT Caffe2Targets ++ DESTINATION ${CMAKE_INSTALL_LIBDIR}) ++ endif() + + if(CAFFE2_COMPILER_SUPPORTS_AVX512_EXTENSIONS) + add_library(Caffe2_perfkernels_avx512 STATIC ${avx512_srcs}) +@@ -84,6 +89,11 @@ if(CAFFE2_COMPILER_SUPPORTS_AVX2_EXTENSIONS) + list(APPEND + Caffe2_DEPENDENCY_WHOLE_LINK_LIBS + "Caffe2_perfkernels_avx512_interface") ++ if(NOT BUILD_SHARED_LIBS) ++ install(TARGETS Caffe2_perfkernels_avx512 ++ EXPORT Caffe2Targets ++ DESTINATION ${CMAKE_INSTALL_LIBDIR}) ++ endif() + endif() + endif() + +diff --git a/modules/detectron/CMakeLists.txt b/modules/detectron/CMakeLists.txt +index 8041e71d35..c7ec50eba5 100644 +--- a/modules/detectron/CMakeLists.txt ++++ b/modules/detectron/CMakeLists.txt +@@ -17,9 +17,12 @@ if(BUILD_CAFFE2_OPS) + + torch_set_target_props(caffe2_detectron_ops_gpu) + target_link_libraries(caffe2_detectron_ops_gpu torch ${OpenMP_link}) +- install(TARGETS caffe2_detectron_ops_gpu DESTINATION lib) ++ install(TARGETS caffe2_detectron_ops_gpu ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + if(MSVC) +- install(FILES $ DESTINATION lib OPTIONAL) ++ install(FILES $ DESTINATION ${CMAKE_INSTALL_BINDIR} OPTIONAL) + endif() + elseif(USE_ROCM) + hip_include_directories(${Caffe2_HIP_INCLUDES}) +@@ -31,7 +34,10 @@ if(BUILD_CAFFE2_OPS) + torch_set_target_props(caffe2_detectron_ops_hip) + target_compile_options(caffe2_detectron_ops_hip PRIVATE ${HIP_CXX_FLAGS}) + target_link_libraries(caffe2_detectron_ops_hip torch) +- install(TARGETS caffe2_detectron_ops_hip DESTINATION lib) ++ install(TARGETS caffe2_detectron_ops_hip ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + elseif(NOT IOS_PLATFORM) + add_library(caffe2_detectron_ops SHARED ${Detectron_CPU_SRCS}) + if(HAVE_SOVERSION) +@@ -40,9 +46,12 @@ if(BUILD_CAFFE2_OPS) + endif() + torch_set_target_props(caffe2_detectron_ops) + target_link_libraries(caffe2_detectron_ops torch ${OpenMP_link}) +- install(TARGETS caffe2_detectron_ops DESTINATION lib) ++ install(TARGETS caffe2_detectron_ops ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + if(MSVC) +- install(FILES $ DESTINATION lib OPTIONAL) ++ install(FILES $ DESTINATION ${CMAKE_INSTALL_BINDIR} OPTIONAL) + endif() + endif() + endif() +diff --git a/modules/observers/CMakeLists.txt b/modules/observers/CMakeLists.txt +index 8796354dc8..378e0af4cf 100644 +--- a/modules/observers/CMakeLists.txt ++++ b/modules/observers/CMakeLists.txt +@@ -21,10 +21,13 @@ endif() + target_link_libraries(caffe2_observers PUBLIC torch_library) + target_include_directories(caffe2_observers PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/..) + target_compile_options(caffe2_observers PRIVATE "-DCAFFE2_BUILD_OBSERVER_LIB") +-install(TARGETS caffe2_observers DESTINATION lib) ++install(TARGETS caffe2_observers ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + caffe2_interface_library(caffe2_observers caffe2_observers_library) + if(MSVC AND BUILD_SHARED_LIBS) +- install(FILES $ DESTINATION lib OPTIONAL) ++ install(FILES $ DESTINATION ${CMAKE_INSTALL_BINDIR} OPTIONAL) + endif() + + if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO) +diff --git a/torch/lib/c10d/CMakeLists.txt b/torch/lib/c10d/CMakeLists.txt +index 4e72e2e32f..87751c779c 100644 +--- a/torch/lib/c10d/CMakeLists.txt ++++ b/torch/lib/c10d/CMakeLists.txt +@@ -150,7 +150,10 @@ endif() + + target_link_libraries(c10d PUBLIC ${C10D_LIBS}) + +-install(TARGETS c10d DESTINATION lib) ++install(TARGETS c10d ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + + option(BUILD_EXAMPLES "Build examples" OFF) + if(BUILD_EXAMPLES) diff --git a/recipes/libtorch/all/patches/0007-unvendor-dependencies-and-honor-flags.patch b/recipes/libtorch/all/patches/0007-unvendor-dependencies-and-honor-flags.patch new file mode 100644 index 00000000000000..66d3c00e2aed7a --- /dev/null +++ b/recipes/libtorch/all/patches/0007-unvendor-dependencies-and-honor-flags.patch @@ -0,0 +1,392 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index ef677702ce..f2e594c81f 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -272,7 +272,7 @@ if(WIN32) + set(USE_TENSORPIPE OFF) + message(WARNING "TensorPipe cannot be used on Windows. Set it to OFF") + +- if(USE_DISTRIBUTED AND NOT DEFINED ENV{libuv_ROOT}) ++ if(0) # don't care, libuv is a dependency of gloo + find_library( + libuv_tmp_LIBRARY + NAMES uv libuv +@@ -538,6 +538,8 @@ endif() + # The below means we are cross compiling for arm64 or x86_64 on MacOSX + if(NOT IOS AND CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_OSX_ARCHITECTURES MATCHES "^(x86_64|arm64)$") + set(CROSS_COMPILING_MACOSX TRUE) ++endif() ++if(0) # trust conan protobuf recipe for cross compilation (anyway it can't work because all the vendoring stuff is missing) + # We need to compile a universal protoc to not fail protobuf build + execute_process(COMMAND ./scripts/build_host_protoc.sh --other-flags "-DCMAKE_OSX_ARCHITECTURES=x86_64;arm64" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} +@@ -617,7 +619,6 @@ endif() + set(CMAKE_C_STANDARD 11) + set(CMAKE_CXX_STANDARD 14) + if(NOT MSVC) +- string(APPEND CMAKE_CXX_FLAGS " -O2 -fPIC") + string(APPEND CMAKE_CXX_FLAGS " -Wno-narrowing") + # Eigen fails to build with some versions, so convert this to a warning + # Details at http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1459 +diff --git a/aten/src/ATen/CMakeLists.txt b/aten/src/ATen/CMakeLists.txt +index 486724d180..eb87a1a2d3 100644 +--- a/aten/src/ATen/CMakeLists.txt ++++ b/aten/src/ATen/CMakeLists.txt +@@ -298,13 +298,6 @@ if(NOT MSVC AND NOT EMSCRIPTEN AND NOT INTERN_BUILD_MOBILE) + list(APPEND ATen_THIRD_PARTY_INCLUDE ${CMAKE_BINARY_DIR}/include) + link_directories(${CMAKE_BINARY_DIR}/sleef/lib) + else() +- add_library(sleef SHARED IMPORTED) +- find_library(SLEEF_LIBRARY sleef) +- if(NOT SLEEF_LIBRARY) +- message(FATAL_ERROR "Cannot find sleef") +- endif() +- message("Found sleef: ${SLEEF_LIBRARY}") +- set_target_properties(sleef PROPERTIES IMPORTED_LOCATION "${SLEEF_LIBRARY}") + endif() + list(APPEND ATen_CPU_DEPENDENCY_LIBS sleef) + +diff --git a/binaries/CMakeLists.txt b/binaries/CMakeLists.txt +index 74df5089e4..c43e395d44 100644 +--- a/binaries/CMakeLists.txt ++++ b/binaries/CMakeLists.txt +@@ -70,7 +70,7 @@ endif() + + if(USE_ZMQ) + caffe2_binary_target("zmq_feeder.cc") +- target_link_libraries(zmq_feeder ${ZMQ_LIBRARIES}) ++ target_link_libraries(zmq_feeder CONAN_PKG::zeromq) + endif() + + if(USE_MPI) +@@ -82,14 +82,14 @@ if(USE_OPENCV AND USE_LEVELDB) + caffe2_binary_target("convert_encoded_to_raw_leveldb.cc") + target_link_libraries( + convert_encoded_to_raw_leveldb +- ${OpenCV_LIBS} ${LevelDB_LIBRARIES} ${Snappy_LIBRARIES}) ++ CONAN_PKG::opencv CONAN_PKG::leveldb) + endif() + + if(USE_OPENCV) + caffe2_binary_target("make_image_db.cc") +- target_link_libraries(make_image_db ${OpenCV_LIBS}) ++ target_link_libraries(make_image_db CONAN_PKG::opencv) + caffe2_binary_target("convert_image_to_tensor.cc") +- target_link_libraries(convert_image_to_tensor ${OpenCV_LIBS}) ++ target_link_libraries(convert_image_to_tensor CONAN_PKG::opencv) + endif() + + if(USE_OBSERVERS) +@@ -98,7 +98,7 @@ endif() + + if(USE_OBSERVERS AND USE_OPENCV) + caffe2_binary_target("convert_and_benchmark.cc") +- target_link_libraries(convert_and_benchmark ${OpenCV_LIBS}) ++ target_link_libraries(convert_and_benchmark CONAN_PKG::opencv) + endif() + + # ---[ tutorials +diff --git a/c10/CMakeLists.txt b/c10/CMakeLists.txt +index a267af43d1..63382626fd 100644 +--- a/c10/CMakeLists.txt ++++ b/c10/CMakeLists.txt +@@ -73,8 +73,7 @@ if(USE_NUMA) + message(STATUS "NUMA paths:") + message(STATUS ${Numa_INCLUDE_DIR}) + message(STATUS ${Numa_LIBRARIES}) +- include_directories(SYSTEM ${Numa_INCLUDE_DIR}) +- target_link_libraries(c10 PRIVATE ${Numa_LIBRARIES}) ++ target_link_libraries(c10 PRIVATE CONAN_PKG::libnuma) + else() + message(STATUS "don't use NUMA") + endif() +diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt +index 8eeb54e653..eacc2c71e7 100644 +--- a/caffe2/CMakeLists.txt ++++ b/caffe2/CMakeLists.txt +@@ -98,7 +98,7 @@ endif() + # Note: the folders that are being commented out have not been properly + # addressed yet. + +-if(NOT MSVC AND USE_XNNPACK) ++if(0) # don't try to build fxdiv from source + if(NOT TARGET fxdiv) + set(FXDIV_BUILD_TESTS OFF CACHE BOOL "") + set(FXDIV_BUILD_BENCHMARKS OFF CACHE BOOL "") +@@ -348,7 +348,7 @@ if(NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE) + # up being empty. Downstream targets should also add a #ifdef guard. + if(NOT WIN32) + add_library(process_group_agent "${TORCH_SRC_DIR}/csrc/distributed/rpc/process_group_agent.cpp" "${TORCH_SRC_DIR}/csrc/distributed/rpc/process_group_agent.h") +- target_link_libraries(process_group_agent PRIVATE torch c10d fmt::fmt-header-only) ++ target_link_libraries(process_group_agent PRIVATE torch c10d CONAN_PKG::fmt) + add_dependencies(process_group_agent torch c10d) + install(TARGETS process_group_agent DESTINATION ${CMAKE_INSTALL_LIBDIR}) + +@@ -359,7 +359,7 @@ if(NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE) + "${TORCH_SRC_DIR}/csrc/distributed/rpc/tensorpipe_utils.cpp" + "${TORCH_SRC_DIR}/csrc/distributed/rpc/tensorpipe_utils.h" + ) +- target_link_libraries(tensorpipe_agent PRIVATE torch c10d tensorpipe fmt::fmt-header-only) ++ target_link_libraries(tensorpipe_agent PRIVATE torch c10d CONAN_PKG::fmt) + add_dependencies(tensorpipe_agent torch c10d) + if(USE_TENSORPIPE) + if(USE_CUDA) +@@ -374,8 +374,7 @@ if(NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE) + endif() + + target_compile_definitions(tensorpipe_agent PUBLIC USE_TENSORPIPE) +- target_link_libraries(tensorpipe_agent PRIVATE tensorpipe) +- add_dependencies(tensorpipe_agent tensorpipe) ++ target_link_libraries(tensorpipe_agent PRIVATE CONAN_PKG::tensorpipe) + install(TARGETS tensorpipe_agent DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() + endif() +diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake +index 72b7d2809f..097f483566 100644 +--- a/cmake/Dependencies.cmake ++++ b/cmake/Dependencies.cmake +@@ -82,7 +82,7 @@ else() + "Cannot find threading library. Caffe2 requires Threads to compile.") + endif() + +-if(USE_TBB) ++if(0) + message(STATUS "Compiling TBB from source") + # Unset our restrictive C++ flags here and reset them later. + # Remove this once we use proper target_compile_options. +@@ -131,8 +131,7 @@ elseif(BLAS STREQUAL "ATLAS") + list(APPEND Caffe2_PUBLIC_DEPENDENCY_LIBS cblas) + elseif(BLAS STREQUAL "OpenBLAS") + find_package(OpenBLAS REQUIRED) +- include_directories(SYSTEM ${OpenBLAS_INCLUDE_DIR}) +- list(APPEND Caffe2_PUBLIC_DEPENDENCY_LIBS ${OpenBLAS_LIB}) ++ list(APPEND Caffe2_PUBLIC_DEPENDENCY_LIBS OpenBLAS::OpenBLAS) + elseif(BLAS STREQUAL "MKL") + if(BLAS_SET_BY_USER) + find_package(MKL REQUIRED) +@@ -410,7 +409,6 @@ if(USE_QNNPACK) + # We build static versions of QNNPACK and pthreadpool but link + # them into a shared library for Caffe2, so they need PIC. + set_property(TARGET qnnpack PROPERTY POSITION_INDEPENDENT_CODE ON) +- set_property(TARGET cpuinfo PROPERTY POSITION_INDEPENDENT_CODE ON) + + if(QNNPACK_CUSTOM_THREADPOOL) + target_compile_definitions( +@@ -495,7 +493,6 @@ endif() + + # ---[ NNPACK + if(USE_NNPACK) +- include(${CMAKE_CURRENT_LIST_DIR}/External/nnpack.cmake) + if(NNPACK_FOUND) + if(TARGET nnpack) + # ---[ NNPACK is being built together with Caffe2: explicitly specify dependency +@@ -542,13 +539,8 @@ if(USE_XNNPACK AND NOT USE_SYSTEM_XNNPACK) + include_directories(SYSTEM ${XNNPACK_INCLUDE_DIR}) + list(APPEND Caffe2_DEPENDENCY_LIBS XNNPACK) + elseif(NOT TARGET XNNPACK AND USE_SYSTEM_XNNPACK) +- add_library(XNNPACK SHARED IMPORTED) +- find_library(XNNPACK_LIBRARY XNNPACK) +- set_property(TARGET XNNPACK PROPERTY IMPORTED_LOCATION "${XNNPACK_LIBRARY}") +- if(NOT XNNPACK_LIBRARY) +- message(FATAL_ERROR "Cannot find XNNPACK") +- endif() +- message("-- Found XNNPACK: ${XNNPACK_LIBRARY}") ++ add_library(XNNPACK INTERFACE IMPORTED) ++ set_property(TARGET XNNPACK PROPERTY INTERFACE_LINK_LIBRARIES CONAN_PKG::xnnpack) + list(APPEND Caffe2_DEPENDENCY_LIBS XNNPACK) + endif() + +@@ -556,9 +548,10 @@ endif() + if(USE_VULKAN) + set(Vulkan_LIBS) + set(Vulkan_INCLUDES) +- include(${CMAKE_CURRENT_LIST_DIR}/VulkanDependencies.cmake) +- list(APPEND Caffe2_DEPENDENCY_LIBS ${Vulkan_LIBS}) +- include_directories(SYSTEM ${Vulkan_INCLUDES}) ++ list(APPEND Caffe2_DEPENDENCY_LIBS CONAN_PKG::vulkan-loader CONAN_PKG::vulkan-headers) ++ if(USE_VULKAN_SHADERC_RUNTIME) ++ list(APPEND Caffe2_DEPENDENCY_LIBS CONAN_PKG::shaderc) ++ endif() + endif() + + # ---[ gflags +@@ -757,13 +750,8 @@ endif() + # ---[ LevelDB + # ---[ Snappy + if(USE_LEVELDB) +- find_package(LevelDB) +- find_package(Snappy) +- if(LEVELDB_FOUND AND SNAPPY_FOUND) +- include_directories(SYSTEM ${LevelDB_INCLUDE}) +- list(APPEND Caffe2_DEPENDENCY_LIBS ${LevelDB_LIBRARIES}) +- include_directories(SYSTEM ${Snappy_INCLUDE_DIR}) +- list(APPEND Caffe2_DEPENDENCY_LIBS ${Snappy_LIBRARIES}) ++ if(1) ++ list(APPEND Caffe2_DEPENDENCY_LIBS CONAN_PKG::leveldb) + else() + message(WARNING "Not compiling with LevelDB. Suppress this warning with -DUSE_LEVELDB=OFF") + caffe2_update_option(USE_LEVELDB OFF) +@@ -773,10 +761,9 @@ endif() + # ---[ NUMA + if(USE_NUMA) + if(LINUX) +- find_package(Numa) ++ set(NUMA_FOUND 1) + if(NUMA_FOUND) +- include_directories(SYSTEM ${Numa_INCLUDE_DIR}) +- list(APPEND Caffe2_DEPENDENCY_LIBS ${Numa_LIBRARIES}) ++ list(APPEND Caffe2_DEPENDENCY_LIBS CONAN_PKG::libnuma) + else() + message(WARNING "Not compiling with NUMA. Suppress this warning with -DUSE_NUMA=OFF") + caffe2_update_option(USE_NUMA OFF) +@@ -789,10 +776,9 @@ endif() + + # ---[ ZMQ + if(USE_ZMQ) +- find_package(ZMQ) ++ set(ZMQ_FOUND 1) + if(ZMQ_FOUND) +- include_directories(SYSTEM ${ZMQ_INCLUDE_DIR}) +- list(APPEND Caffe2_DEPENDENCY_LIBS ${ZMQ_LIBRARIES}) ++ list(APPEND Caffe2_DEPENDENCY_LIBS CONAN_PKG::zeromq) + else() + message(WARNING "Not compiling with ZMQ. Suppress this warning with -DUSE_ZMQ=OFF") + caffe2_update_option(USE_ZMQ OFF) +@@ -801,10 +787,9 @@ endif() + + # ---[ Redis + if(USE_REDIS) +- find_package(Hiredis) +- if(HIREDIS_FOUND) +- include_directories(SYSTEM ${Hiredis_INCLUDE}) +- list(APPEND Caffe2_DEPENDENCY_LIBS ${Hiredis_LIBRARIES}) ++ find_package(hiredis) ++ if(hiredis_FOUND) ++ list(APPEND Caffe2_DEPENDENCY_LIBS hiredis::hiredis) + else() + message(WARNING "Not compiling with Redis. Suppress this warning with -DUSE_REDIS=OFF") + caffe2_update_option(USE_REDIS OFF) +@@ -825,10 +810,9 @@ if(USE_OPENCV) + endif() + endif() + if(OpenCV_FOUND) +- include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS}) +- list(APPEND Caffe2_DEPENDENCY_LIBS ${OpenCV_LIBS}) ++ list(APPEND Caffe2_DEPENDENCY_LIBS CONAN_PKG::opencv) + if(MSVC AND USE_CUDA) +- list(APPEND Caffe2_CUDA_DEPENDENCY_LIBS ${OpenCV_LIBS}) ++ list(APPEND Caffe2_CUDA_DEPENDENCY_LIBS CONAN_PKG::opencv) + endif() + message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})") + else() +@@ -1028,8 +1012,7 @@ if(USE_MPI) + message(STATUS "MPI include path: " ${MPI_CXX_INCLUDE_PATH}) + message(STATUS "MPI LINK flags path: " ${MPI_CXX_LINK_FLAGS}) + message(STATUS "MPI libraries: " ${MPI_CXX_LIBRARIES}) +- include_directories(SYSTEM ${MPI_CXX_INCLUDE_PATH}) +- list(APPEND Caffe2_DEPENDENCY_LIBS ${MPI_CXX_LIBRARIES}) ++ list(APPEND Caffe2_DEPENDENCY_LIBS CONAN_PKG::openmpi) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${MPI_CXX_LINK_FLAGS}") + find_program(OMPI_INFO + NAMES ompi_info +@@ -1354,9 +1337,8 @@ if(USE_DISTRIBUTED AND USE_TENSORPIPE) + set(TP_BUILD_LIBUV ON CACHE BOOL "" FORCE) + set(TP_STATIC_OR_SHARED STATIC CACHE STRING "" FORCE) + +- add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/tensorpipe) + +- list(APPEND Caffe2_DEPENDENCY_LIBS tensorpipe) ++ list(APPEND Caffe2_DEPENDENCY_LIBS CONAN_PKG::tensorpipe) + endif() + endif() + +@@ -1406,10 +1388,7 @@ if(NOT INTERN_BUILD_MOBILE AND BUILD_CAFFE2_OPS) + endif() + + if(USE_ZSTD) +- list(APPEND Caffe2_DEPENDENCY_LIBS libzstd_static) +- include_directories(SYSTEM ${CMAKE_CURRENT_LIST_DIR}/../third_party/zstd/lib) +- add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/zstd/build/cmake) +- set_property(TARGET libzstd_static PROPERTY POSITION_INDEPENDENT_CODE ON) ++ list(APPEND Caffe2_DEPENDENCY_LIBS CONAN_PKG::zstd) + endif() + + # ---[ Onnx +@@ -1436,7 +1415,6 @@ if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO AND NOT INTERN_DISABLE_ONNX) + if(NOT USE_SYSTEM_ONNX) + add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/onnx EXCLUDE_FROM_ALL) + endif() +- add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/foxi EXCLUDE_FROM_ALL) + + add_definitions(-DONNX_NAMESPACE=${ONNX_NAMESPACE}) + if(NOT USE_SYSTEM_ONNX) +@@ -1450,23 +1428,11 @@ if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO AND NOT INTERN_DISABLE_ONNX) + endif() + list(APPEND Caffe2_DEPENDENCY_WHOLE_LINK_LIBS onnx_library) + else() +- add_library(onnx SHARED IMPORTED) +- find_library(ONNX_LIBRARY onnx) +- if(NOT ONNX_LIBRARY) +- message(FATAL_ERROR "Cannot find onnx") +- endif() +- set_property(TARGET onnx PROPERTY IMPORTED_LOCATION ${ONNX_LIBRARY}) +- add_library(onnx_proto SHARED IMPORTED) +- find_library(ONNX_PROTO_LIBRARY onnx_proto) +- if(NOT ONNX_PROTO_LIBRARY) +- message(FATAL_ERROR "Cannot find onnx") +- endif() +- set_property(TARGET onnx_proto PROPERTY IMPORTED_LOCATION ${ONNX_PROTO_LIBRARY}) +- message("-- Found onnx: ${ONNX_LIBRARY} ${ONNX_PROTO_LIBRARY}") ++ find_package(ONNX REQUIRED CONFIG) + list(APPEND Caffe2_DEPENDENCY_LIBS onnx_proto onnx) + endif() + include_directories(${FOXI_INCLUDE_DIRS}) +- list(APPEND Caffe2_DEPENDENCY_LIBS foxi_loader) ++ list(APPEND Caffe2_DEPENDENCY_LIBS CONAN_PKG::foxi) + # Recover the build shared libs option. + set(BUILD_SHARED_LIBS ${TEMP_BUILD_SHARED_LIBS}) + endif() +@@ -1666,8 +1632,9 @@ if(NOT INTERN_BUILD_MOBILE) + CACHE BOOL "Copy the required BLAS DLLs into the TH install dirs") + endif() + +- find_package(LAPACK) +- if(LAPACK_FOUND) ++ option(USE_LAPACK "Use LAPACK" ON) ++ if(USE_LAPACK) ++ find_package(LAPACK REQUIRED) + set(USE_LAPACK 1) + list(APPEND Caffe2_PRIVATE_DEPENDENCY_LIBS ${LAPACK_LIBRARIES}) + else() +@@ -1781,7 +1748,6 @@ endif() + # + set(TEMP_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) + set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libs" FORCE) +-add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/fmt) + + # Disable compiler feature checks for `fmt`. + # +@@ -1790,9 +1756,8 @@ add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/fmt) + # CMAKE_CXX_FLAGS in ways that break feature checks. Since we already know + # `fmt` is compatible with a superset of the compilers that PyTorch is, it + # shouldn't be too bad to just disable the checks. +-set_target_properties(fmt-header-only PROPERTIES INTERFACE_COMPILE_FEATURES "") + +-list(APPEND Caffe2_DEPENDENCY_LIBS fmt::fmt-header-only) ++list(APPEND Caffe2_DEPENDENCY_LIBS CONAN_PKG::fmt) + set(BUILD_SHARED_LIBS ${TEMP_BUILD_SHARED_LIBS} CACHE BOOL "Build shared libs" FORCE) + + # ---[ Kineto +diff --git a/torch/CMakeLists.txt b/torch/CMakeLists.txt +index f15d32b592..b07604ebc2 100644 +--- a/torch/CMakeLists.txt ++++ b/torch/CMakeLists.txt +@@ -272,7 +272,7 @@ if(USE_DISTRIBUTED) + set_source_files_properties(${TORCH_SRC_DIR}/csrc/distributed/c10d/init.cpp PROPERTIES COMPILE_FLAGS "-Wno-cast-function-type") + endif() + if(USE_TENSORPIPE) +- list(APPEND TORCH_PYTHON_LINK_LIBRARIES tensorpipe) ++ list(APPEND TORCH_PYTHON_LINK_LIBRARIES CONAN_PKG::tensorpipe) + list(APPEND TORCH_PYTHON_PUBLIC_COMPILE_DEFINITIONS USE_TENSORPIPE) + endif() + list(APPEND TORCH_PYTHON_LINK_LIBRARIES c10d) diff --git a/recipes/libtorch/all/test_package/CMakeLists.txt b/recipes/libtorch/all/test_package/CMakeLists.txt new file mode 100644 index 00000000000000..b7c8000a73fec1 --- /dev/null +++ b/recipes/libtorch/all/test_package/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.1) +project(test_package) + +include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) +conan_basic_setup(TARGETS) + +find_package(Torch REQUIRED CONFIG) + +add_executable(${PROJECT_NAME} test_package.cpp) +target_link_libraries(${PROJECT_NAME} ${TORCH_LIBRARIES}) +set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 14) diff --git a/recipes/libtorch/all/test_package/conanfile.py b/recipes/libtorch/all/test_package/conanfile.py new file mode 100644 index 00000000000000..a9f777f7680ff1 --- /dev/null +++ b/recipes/libtorch/all/test_package/conanfile.py @@ -0,0 +1,17 @@ +from conans import ConanFile, CMake, tools +import os + + +class TestPackageConan(ConanFile): + settings = "os", "arch", "compiler", "build_type" + generators = "cmake", "cmake_find_package_multi" + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def test(self): + if not tools.cross_building(self.settings): + bin_path = os.path.join("bin", "test_package") + self.run(bin_path, run_environment=True) diff --git a/recipes/libtorch/all/test_package/test_package.cpp b/recipes/libtorch/all/test_package/test_package.cpp new file mode 100644 index 00000000000000..ca31e23db7a821 --- /dev/null +++ b/recipes/libtorch/all/test_package/test_package.cpp @@ -0,0 +1,31 @@ +#include + +#include + +struct Net: torch::nn::Module { + Net(int64_t N, int64_t M): linear(register_module("linear", torch::nn::Linear(N, M))) { + another_bias = register_parameter("b", torch::randn(M)); + } + + torch::Tensor forward(torch::Tensor input) { + return linear(input) + another_bias; + } + + torch::nn::Linear linear; + torch::Tensor another_bias; +}; + +int main() { + torch::Tensor tensor = torch::eye(3); + std::cout << tensor << std::endl; + + Net net(4, 5); + for (const auto& p : net.parameters()) { + std::cout << p << std::endl; + } + for (const auto& pair : net.named_parameters()) { + std::cout << pair.key() << ": " << pair.value() << std::endl; + } + + return 0; +} diff --git a/recipes/libtorch/config.yml b/recipes/libtorch/config.yml new file mode 100644 index 00000000000000..c9ce70eeda37d0 --- /dev/null +++ b/recipes/libtorch/config.yml @@ -0,0 +1,3 @@ +versions: + "1.8.1": + folder: all