From 3c924b233859f9edb0028188a66920f4392c9344 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 20 Jun 2024 18:31:03 -0400 Subject: [PATCH 01/35] automatically set pytorch requires Signed-off-by: Jinzhe Zeng (cherry picked from commit ebb1a873bc2cdc48bf3fac3193741123c35c8bfd) Signed-off-by: Jinzhe Zeng --- backend/dp_backend.py | 15 +++++++++++++-- backend/find_pytorch.py | 40 +++++++++++++++++++++++++++++++++++++--- backend/read_env.py | 2 +- pyproject.toml | 3 --- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/backend/dp_backend.py b/backend/dp_backend.py index 2ca0ff2f93..dbd2d2a52b 100644 --- a/backend/dp_backend.py +++ b/backend/dp_backend.py @@ -7,6 +7,9 @@ from scikit_build_core import build as _orig +from .find_pytorch import ( + find_pytorch, +) from .find_tensorflow import ( find_tensorflow, ) @@ -40,10 +43,18 @@ def __dir__() -> List[str]: def get_requires_for_build_wheel( config_settings: dict, ) -> List[str]: - return _orig.get_requires_for_build_wheel(config_settings) + find_tensorflow()[1] + return ( + _orig.get_requires_for_build_wheel(config_settings) + + find_tensorflow()[1] + + find_pytorch()[1] + ) def get_requires_for_build_editable( config_settings: dict, ) -> List[str]: - return _orig.get_requires_for_build_editable(config_settings) + find_tensorflow()[1] + return ( + _orig.get_requires_for_build_editable(config_settings) + + find_tensorflow()[1] + + find_pytorch()[1] + ) diff --git a/backend/find_pytorch.py b/backend/find_pytorch.py index f039b6f289..f93090fe51 100644 --- a/backend/find_pytorch.py +++ b/backend/find_pytorch.py @@ -17,12 +17,14 @@ get_path, ) from typing import ( + List, Optional, + Tuple, ) @lru_cache -def find_pytorch() -> Optional[str]: +def find_pytorch() -> Tuple[Optional[str], List[str]]: """Find PyTorch library. Tries to find PyTorch in the order of: @@ -39,9 +41,12 @@ def find_pytorch() -> Optional[str]: ------- str, optional PyTorch library path if found. + list of str + TensorFlow requirement if not found. Empty if found. """ if os.environ.get("DP_ENABLE_PYTORCH", "0") == "0": - return None + return None, [] + requires = [] pt_spec = None if (pt_spec is None or not pt_spec) and os.environ.get("PYTORCH_ROOT") is not None: @@ -73,4 +78,33 @@ def find_pytorch() -> Optional[str]: # IndexError if submodule_search_locations is an empty list except (AttributeError, TypeError, IndexError): pt_install_dir = None - return pt_install_dir + requires.extend(get_pt_requirement()["torch"]) + return pt_install_dir, requires + + +@lru_cache +def get_pt_requirement(pt_version: str = "") -> dict: + """Get PyTorch requirement when PT is not installed. + + If pt_version is not given and the environment variable `PYTORCH_VERSION` is set, use it as the requirement. + + Parameters + ---------- + pt_version : str, optional + PT version + + Returns + ------- + dict + PyTorch requirement. + """ + if pt_version is None: + return {"torch": []} + if pt_version == "": + pt_version = os.environ.get("PYTORCH_VERSION", "") + + return { + "torch": [ + f"torch=={pt_version}" if pt_version != "" else "torch>=2a", + ], + } diff --git a/backend/read_env.py b/backend/read_env.py index 14935dcc0f..3031820b3e 100644 --- a/backend/read_env.py +++ b/backend/read_env.py @@ -103,7 +103,7 @@ def get_argument_from_env() -> Tuple[str, list, list, dict, str]: tf_version = None if os.environ.get("DP_ENABLE_PYTORCH", "0") == "1": - pt_install_dir = find_pytorch() + pt_install_dir, _ = find_pytorch() if pt_install_dir is None: raise RuntimeError("Cannot find installed PyTorch.") cmake_args.extend( diff --git a/pyproject.toml b/pyproject.toml index 2cb489ce43..39ade063b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,9 +126,6 @@ cu12 = [ "nvidia-cudnn-cu12<9", "nvidia-cuda-nvcc-cu12", ] -torch = [ - "torch>=2a", -] [tool.deepmd_build_backend.scripts] dp = "deepmd.main:main" From 63c1bd79c0029a0d2f78e7b4c4e18de06c15825a Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 20 Jun 2024 18:44:53 -0400 Subject: [PATCH 02/35] remove the error message when PT is not preinstalled Signed-off-by: Jinzhe Zeng (cherry picked from commit 60b690392b8b5ed7c9c8af811046490dcba24380) --- backend/read_env.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/read_env.py b/backend/read_env.py index 3031820b3e..53f0ca7d28 100644 --- a/backend/read_env.py +++ b/backend/read_env.py @@ -104,8 +104,6 @@ def get_argument_from_env() -> Tuple[str, list, list, dict, str]: if os.environ.get("DP_ENABLE_PYTORCH", "0") == "1": pt_install_dir, _ = find_pytorch() - if pt_install_dir is None: - raise RuntimeError("Cannot find installed PyTorch.") cmake_args.extend( [ "-DENABLE_PYTORCH=ON", From 59f36afeed5351a8d6667bf9fe5e4713de3dc7b4 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 17:27:35 -0400 Subject: [PATCH 03/35] pass pt version Signed-off-by: Jinzhe Zeng --- backend/dynamic_metadata.py | 8 +++++++- backend/find_pytorch.py | 25 +++++++++++++++++++++++++ backend/read_env.py | 8 +++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/backend/dynamic_metadata.py b/backend/dynamic_metadata.py index 2a66ff065c..83123e6e41 100644 --- a/backend/dynamic_metadata.py +++ b/backend/dynamic_metadata.py @@ -9,6 +9,9 @@ Optional, ) +from .find_pytorch import ( + get_pt_requirement, +) from .find_tensorflow import ( get_tf_requirement, ) @@ -33,7 +36,9 @@ def dynamic_metadata( settings: Optional[Dict[str, object]] = None, ): assert field in ["optional-dependencies", "entry-points", "scripts"] - _, _, find_libpython_requires, extra_scripts, tf_version = get_argument_from_env() + _, _, find_libpython_requires, extra_scripts, tf_version, pt_version = ( + get_argument_from_env() + ) with Path("pyproject.toml").open("rb") as f: pyproject = tomllib.load(f) @@ -51,4 +56,5 @@ def dynamic_metadata( return { **optional_dependencies, **get_tf_requirement(tf_version), + **get_pt_requirement(pt_version), } diff --git a/backend/find_pytorch.py b/backend/find_pytorch.py index f93090fe51..82d8f8bf71 100644 --- a/backend/find_pytorch.py +++ b/backend/find_pytorch.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: LGPL-3.0-or-later +import importlib import os import site from functools import ( @@ -20,6 +21,7 @@ List, Optional, Tuple, + Union, ) @@ -108,3 +110,26 @@ def get_pt_requirement(pt_version: str = "") -> dict: f"torch=={pt_version}" if pt_version != "" else "torch>=2a", ], } + + +@lru_cache +def get_pt_version(pt_path: Union[str, Path]) -> str: + """Get TF version from a TF Python library path. + + Parameters + ---------- + pt_path : str or Path + PT Python library path + + Returns + ------- + str + version + """ + if pt_path is None or pt_path == "": + return "" + version_file = Path(pt_path) / "version.py" + spec = importlib.util.spec_from_file_location("torch.version", version_file) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.__version__ diff --git a/backend/read_env.py b/backend/read_env.py index 53f0ca7d28..4e43e0f70b 100644 --- a/backend/read_env.py +++ b/backend/read_env.py @@ -15,6 +15,7 @@ from .find_pytorch import ( find_pytorch, + get_pt_version, ) from .find_tensorflow import ( find_tensorflow, @@ -23,7 +24,7 @@ @lru_cache -def get_argument_from_env() -> Tuple[str, list, list, dict, str]: +def get_argument_from_env() -> Tuple[str, list, list, dict, str, str]: """Get the arguments from environment variables. The environment variables are assumed to be not changed during the build. @@ -40,6 +41,8 @@ def get_argument_from_env() -> Tuple[str, list, list, dict, str]: The extra scripts to be installed. str The TensorFlow version. + str + The PyTorch version. """ cmake_args = [] extra_scripts = {} @@ -104,6 +107,7 @@ def get_argument_from_env() -> Tuple[str, list, list, dict, str]: if os.environ.get("DP_ENABLE_PYTORCH", "0") == "1": pt_install_dir, _ = find_pytorch() + pt_version = get_pt_version(pt_install_dir) cmake_args.extend( [ "-DENABLE_PYTORCH=ON", @@ -112,6 +116,7 @@ def get_argument_from_env() -> Tuple[str, list, list, dict, str]: ) else: cmake_args.append("-DENABLE_PYTORCH=OFF") + pt_version = None cmake_args = [ "-DBUILD_PY_IF:BOOL=TRUE", @@ -123,6 +128,7 @@ def get_argument_from_env() -> Tuple[str, list, list, dict, str]: find_libpython_requires, extra_scripts, tf_version, + pt_version, ) From 84cf0f59c374891c849d73f71217e3a4e954c0f2 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 17:33:01 -0400 Subject: [PATCH 04/35] enable pytorch in cibuildwheel Signed-off-by: Jinzhe Zeng --- pyproject.toml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 39ade063b4..16e9509498 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -208,12 +208,17 @@ manylinux-x86_64-image = "quay.io/pypa/manylinux_2_28_x86_64:2022-11-19-1b19e81" manylinux-aarch64-image = "manylinux_2_28" [tool.cibuildwheel.macos] -environment = { PIP_PREFER_BINARY="1", DP_LAMMPS_VERSION="stable_2Aug2023_update3", DP_ENABLE_IPI="1" } before-all = [ """brew install mpich""", ] repair-wheel-command = """delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} --ignore-missing-dependencies""" +[tool.cibuildwheel.macos.environment] +PIP_PREFER_BINARY = "1" +DP_LAMMPS_VERSION = "stable_2Aug2023_update3" +DP_ENABLE_IPI = "1" +DP_ENABLE_PYTORCH = "1" + [tool.cibuildwheel.linux] repair-wheel-command = "auditwheel repair --exclude libtensorflow_framework.so.2 --exclude libtensorflow_framework.so.1 --exclude libtensorflow_framework.so --exclude _pywrap_tensorflow_internal.so --exclude libtensorflow_cc.so.2 -w {dest_dir} {wheel}" environment-pass = [ @@ -223,7 +228,6 @@ environment-pass = [ "DP_PKG_NAME", "SETUPTOOLS_SCM_PRETEND_VERSION", ] -environment = { PIP_PREFER_BINARY="1", DP_LAMMPS_VERSION="stable_2Aug2023_update3", DP_ENABLE_IPI="1", MPI_HOME="/usr/lib64/mpich", PATH="/usr/lib64/mpich/bin:$PATH" } before-all = [ """if [ ! -z "${DP_PKG_NAME}" ]; then sed -i "s/name = \\"deepmd-kit\\"/name = \\"${DP_PKG_NAME}\\"/g" pyproject.toml; fi""", # https://almalinux.org/blog/2023-12-20-almalinux-8-key-update/ @@ -237,14 +241,23 @@ before-build = [ # old build doesn't support uv """{ if [ "$(uname -m)" = "x86_64" ] ; then uv pip install --system -U build; fi }""", ] +[tool.cibuildwheel.linux.environment] +PIP_PREFER_BINARY = "1" +DP_LAMMPS_VERSION = "stable_2Aug2023_update3" +DP_ENABLE_IPI = "1" +DP_ENABLE_PYTORCH = "1" +MPI_HOME = "/usr/lib64/mpich" +PATH = "/usr/lib64/mpich/bin:$PATH" [tool.cibuildwheel.windows] -environment = { PIP_PREFER_BINARY="1" } test-extras = ["cpu"] test-command = [ "python -m deepmd -h", "dp -h", ] +[tool.cibuildwheel.windows.environment] +PIP_PREFER_BINARY = "1" +DP_ENABLE_PYTORCH = "1" # One can run `tox` or `tox -e gpu` # to run pytest in an isolated environment From 8286b9e5ecf1107203ba74fb23d2cbacda5e0912 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 17:38:13 -0400 Subject: [PATCH 05/35] fix argument Signed-off-by: Jinzhe Zeng --- backend/read_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/read_env.py b/backend/read_env.py index 4e43e0f70b..c3fe2d5127 100644 --- a/backend/read_env.py +++ b/backend/read_env.py @@ -134,6 +134,6 @@ def get_argument_from_env() -> Tuple[str, list, list, dict, str, str]: def set_scikit_build_env(): """Set scikit-build environment variables before executing scikit-build.""" - cmake_minimum_required_version, cmake_args, _, _, _ = get_argument_from_env() + cmake_minimum_required_version, cmake_args, _, _, _, _ = get_argument_from_env() os.environ["SKBUILD_CMAKE_MINIMUM_VERSION"] = cmake_minimum_required_version os.environ["SKBUILD_CMAKE_ARGS"] = ";".join(cmake_args) From 4c28b7570b606d6b532e9cc3d99d9b59044fdfde Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 17:50:31 -0400 Subject: [PATCH 06/35] fix build on macos/windows Signed-off-by: Jinzhe Zeng --- source/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 3a357e1321..6a3103616e 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -219,6 +219,9 @@ if(ENABLE_PYTORCH AND NOT DEEPMD_C_ROOT) set(OP_CXX_ABI ${CMAKE_MATCH_1}) add_definitions(-D_GLIBCXX_USE_CXX11_ABI=${OP_CXX_ABI}) endif() + else() + # Maybe in macos/windows + set(OP_CXX_ABI_PT ${OP_CXX_ABI}) endif() # get torch directory get the directory of the target "torch" get_target_property(_TORCH_LOCATION torch LOCATION) From e270adb37af45d6183fa73e51a5d944da4c965a1 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 18:04:57 -0400 Subject: [PATCH 07/35] use CPU version for build Signed-off-by: Jinzhe Zeng --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 16e9509498..42d7548b1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -248,6 +248,10 @@ DP_ENABLE_IPI = "1" DP_ENABLE_PYTORCH = "1" MPI_HOME = "/usr/lib64/mpich" PATH = "/usr/lib64/mpich/bin:$PATH" +# use CPU version of torch for building, which should also work for GPU +# note: uv has different behavior from pip on extra index url +# https://github.com/astral-sh/uv/blob/main/PIP_COMPATIBILITY.md#packages-that-exist-on-multiple-indexes +UV_EXTRA_INDEX_URL = "https://download.pytorch.org/whl/cpu" [tool.cibuildwheel.windows] test-extras = ["cpu"] From 4e766d3f9a748ff1c91577fa3bc2c57bfdffca5c Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 18:19:38 -0400 Subject: [PATCH 08/35] fix macosx_x86_64 error Signed-off-by: Jinzhe Zeng --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 42d7548b1f..e764c21d31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -219,6 +219,11 @@ DP_LAMMPS_VERSION = "stable_2Aug2023_update3" DP_ENABLE_IPI = "1" DP_ENABLE_PYTORCH = "1" +[[tool.cibuildwheel.overrides]] +# error: 'value' is unavailable: introduced in macOS 10.13 +select = "*-macosx_x86_64" +environment = { MACOSX_DEPLOYMENT_TARGET = "10.13" } + [tool.cibuildwheel.linux] repair-wheel-command = "auditwheel repair --exclude libtensorflow_framework.so.2 --exclude libtensorflow_framework.so.1 --exclude libtensorflow_framework.so --exclude _pywrap_tensorflow_internal.so --exclude libtensorflow_cc.so.2 -w {dest_dir} {wheel}" environment-pass = [ From 14a87a0a6aad8b4c81b2164272e2fb0f268a8d57 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 18:20:22 -0400 Subject: [PATCH 09/35] exclude libc10 and libtorch Signed-off-by: Jinzhe Zeng --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e764c21d31..78d3e808d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,7 +225,7 @@ select = "*-macosx_x86_64" environment = { MACOSX_DEPLOYMENT_TARGET = "10.13" } [tool.cibuildwheel.linux] -repair-wheel-command = "auditwheel repair --exclude libtensorflow_framework.so.2 --exclude libtensorflow_framework.so.1 --exclude libtensorflow_framework.so --exclude _pywrap_tensorflow_internal.so --exclude libtensorflow_cc.so.2 -w {dest_dir} {wheel}" +repair-wheel-command = "auditwheel repair --exclude libtensorflow_framework.so.2 --exclude libtensorflow_framework.so.1 --exclude libtensorflow_framework.so --exclude _pywrap_tensorflow_internal.so --exclude libtensorflow_cc.so.2 --exclude libc10.so --exclude libtorch.so -w {dest_dir} {wheel}" environment-pass = [ "CIBW_BUILD", "DP_VARIANT", From 76225f0e74e36b39d87310e1e385d4b334ece355 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 18:31:53 -0400 Subject: [PATCH 10/35] fix openmp issue on macos-arm64 Signed-off-by: Jinzhe Zeng --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 78d3e808d3..8f880e0c4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -218,6 +218,8 @@ PIP_PREFER_BINARY = "1" DP_LAMMPS_VERSION = "stable_2Aug2023_update3" DP_ENABLE_IPI = "1" DP_ENABLE_PYTORCH = "1" +# for unclear reason, when enabling PyTorch, OpenMP is found accidentally +CMAKE_DISABLE_FIND_PACKAGE_OpenMP = "1" [[tool.cibuildwheel.overrides]] # error: 'value' is unavailable: introduced in macOS 10.13 From 9675b4ea99881777436c1b54958fa80686bf0154 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 18:45:38 -0400 Subject: [PATCH 11/35] exclude libtorch_cpu.so Signed-off-by: Jinzhe Zeng --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8f880e0c4e..906943e2a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -227,7 +227,7 @@ select = "*-macosx_x86_64" environment = { MACOSX_DEPLOYMENT_TARGET = "10.13" } [tool.cibuildwheel.linux] -repair-wheel-command = "auditwheel repair --exclude libtensorflow_framework.so.2 --exclude libtensorflow_framework.so.1 --exclude libtensorflow_framework.so --exclude _pywrap_tensorflow_internal.so --exclude libtensorflow_cc.so.2 --exclude libc10.so --exclude libtorch.so -w {dest_dir} {wheel}" +repair-wheel-command = "auditwheel repair --exclude libtensorflow_framework.so.2 --exclude libtensorflow_framework.so.1 --exclude libtensorflow_framework.so --exclude _pywrap_tensorflow_internal.so --exclude libtensorflow_cc.so.2 --exclude libc10.so --exclude libtorch.so --exclude libtorch_cpu.so -w {dest_dir} {wheel}" environment-pass = [ "CIBW_BUILD", "DP_VARIANT", From d1f47fd0d8d27a2f0f833ba92527b03596ff34c1 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 18:49:17 -0400 Subject: [PATCH 12/35] fix CMAKE_ARGS Signed-off-by: Jinzhe Zeng --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 906943e2a4..229de7ff17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -219,7 +219,7 @@ DP_LAMMPS_VERSION = "stable_2Aug2023_update3" DP_ENABLE_IPI = "1" DP_ENABLE_PYTORCH = "1" # for unclear reason, when enabling PyTorch, OpenMP is found accidentally -CMAKE_DISABLE_FIND_PACKAGE_OpenMP = "1" +CMAKE_ARGS = "-DCMAKE_DISABLE_FIND_PACKAGE_OpenMP=1" [[tool.cibuildwheel.overrides]] # error: 'value' is unavailable: introduced in macOS 10.13 From 13404aab37593cce1939c63ae95e75096ed2d68b Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 18:51:00 -0400 Subject: [PATCH 13/35] inherit env Signed-off-by: Jinzhe Zeng --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 229de7ff17..57e7ebd3b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -224,7 +224,8 @@ CMAKE_ARGS = "-DCMAKE_DISABLE_FIND_PACKAGE_OpenMP=1" [[tool.cibuildwheel.overrides]] # error: 'value' is unavailable: introduced in macOS 10.13 select = "*-macosx_x86_64" -environment = { MACOSX_DEPLOYMENT_TARGET = "10.13" } +inherit.environment = "append" +environment.MACOSX_DEPLOYMENT_TARGET = "10.13" [tool.cibuildwheel.linux] repair-wheel-command = "auditwheel repair --exclude libtensorflow_framework.so.2 --exclude libtensorflow_framework.so.1 --exclude libtensorflow_framework.so --exclude _pywrap_tensorflow_internal.so --exclude libtensorflow_cc.so.2 --exclude libc10.so --exclude libtorch.so --exclude libtorch_cpu.so -w {dest_dir} {wheel}" From 2e91ce8bd1ed8701c88d434a0f9865d8ea133845 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 19:11:17 -0400 Subject: [PATCH 14/35] mpich Signed-off-by: Jinzhe Zeng --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 57e7ebd3b7..48437b3c3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -209,7 +209,7 @@ manylinux-aarch64-image = "manylinux_2_28" [tool.cibuildwheel.macos] before-all = [ - """brew install mpich""", + '''pip install -i https://pypi.anaconda.org/mpi4py/simple mpich''', ] repair-wheel-command = """delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} --ignore-missing-dependencies""" @@ -241,7 +241,7 @@ before-all = [ # https://almalinux.org/blog/2023-12-20-almalinux-8-key-update/ """rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux""", """{ if [ "$(uname -m)" = "x86_64" ] ; then yum config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo && yum install -y cuda-nvcc-${CUDA_VERSION/./-} cuda-cudart-devel-${CUDA_VERSION/./-}; fi }""", - "yum install -y mpich-devel", + '''/opt/python/cp311-cp311/bin/python -m pip install -i https://pypi.anaconda.org/mpi4py/simple mpich''', # uv is not available in the old manylinux image """{ if [ "$(uname -m)" = "x86_64" ] ; then pipx install uv; fi }""", ] @@ -260,6 +260,8 @@ PATH = "/usr/lib64/mpich/bin:$PATH" # note: uv has different behavior from pip on extra index url # https://github.com/astral-sh/uv/blob/main/PIP_COMPATIBILITY.md#packages-that-exist-on-multiple-indexes UV_EXTRA_INDEX_URL = "https://download.pytorch.org/whl/cpu" +# trick to find the correction version of mpich +CMAKE_PREFIX_PATH="/opt/python/cp311-cp311/" [tool.cibuildwheel.windows] test-extras = ["cpu"] From ced2562d7f15ef3278bb9a13afdd7ab6cab5f69f Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 19:35:31 -0400 Subject: [PATCH 15/35] set pt dir to LD_LIBRARY_PATH Signed-off-by: Jinzhe Zeng --- deepmd/tf/lmp.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deepmd/tf/lmp.py b/deepmd/tf/lmp.py index b2e47308ed..e378beecfe 100644 --- a/deepmd/tf/lmp.py +++ b/deepmd/tf/lmp.py @@ -6,6 +6,9 @@ from importlib import ( import_module, ) +from importlib.util import ( + find_spec, +) from pathlib import ( Path, ) @@ -77,6 +80,11 @@ def get_library_path(module: str, filename: str) -> List[str]: tf_dir = tf.sysconfig.get_lib() op_dir = str(SHARED_LIB_DIR) +pt_spec = find_spec("torch") +if pt_spec is not None: + pt_dir = pt_spec.submodule_search_locations[0] +else: + pt_dir = None cuda_library_paths = [] @@ -106,6 +114,7 @@ def get_library_path(module: str, filename: str) -> List[str]: [ os.environ.get(lib_env), tf_dir, + pt_dir, os.path.join(tf_dir, "python"), op_dir, ] From 30b5377ab0aa434c051e8ac4cf8d2f3cbe4b1d3c Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 19:54:08 -0400 Subject: [PATCH 16/35] fix typo Signed-off-by: Jinzhe Zeng --- deepmd/tf/lmp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepmd/tf/lmp.py b/deepmd/tf/lmp.py index e378beecfe..1c3c265db2 100644 --- a/deepmd/tf/lmp.py +++ b/deepmd/tf/lmp.py @@ -80,7 +80,7 @@ def get_library_path(module: str, filename: str) -> List[str]: tf_dir = tf.sysconfig.get_lib() op_dir = str(SHARED_LIB_DIR) -pt_spec = find_spec("torch") +pt_spec = find_spec("torch.lib") if pt_spec is not None: pt_dir = pt_spec.submodule_search_locations[0] else: From f9d4e522b8937e33edc9743e10a1cb1a93f7c6fc Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 21:44:36 -0400 Subject: [PATCH 17/35] prevent OP_CXX_ABI not defined Signed-off-by: Jinzhe Zeng (cherry picked from commit bd7da0bf5ff48507ede5548ecb75a5c9a8631ba4) --- source/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 6a3103616e..cf516b9755 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -221,7 +221,10 @@ if(ENABLE_PYTORCH AND NOT DEEPMD_C_ROOT) endif() else() # Maybe in macos/windows - set(OP_CXX_ABI_PT ${OP_CXX_ABI}) + if(NOT DEFINED OP_CXX_ABI) + set(OP_CXX_ABI 0) + endif() + set(OP_CXX_ABI_PT "${OP_CXX_ABI}") endif() # get torch directory get the directory of the target "torch" get_target_property(_TORCH_LOCATION torch LOCATION) From 1ab3f923affb291b172062b770429618afe1e435 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 22:33:21 -0400 Subject: [PATCH 18/35] Revert "prevent OP_CXX_ABI not defined" This reverts commit f9d4e522b8937e33edc9743e10a1cb1a93f7c6fc. --- source/CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index cf516b9755..6a3103616e 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -221,10 +221,7 @@ if(ENABLE_PYTORCH AND NOT DEEPMD_C_ROOT) endif() else() # Maybe in macos/windows - if(NOT DEFINED OP_CXX_ABI) - set(OP_CXX_ABI 0) - endif() - set(OP_CXX_ABI_PT "${OP_CXX_ABI}") + set(OP_CXX_ABI_PT ${OP_CXX_ABI}) endif() # get torch directory get the directory of the target "torch" get_target_property(_TORCH_LOCATION torch LOCATION) From 55f04fb1a8cfe92f108b0e3f39a5ee05ebed10de Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 22:33:53 -0400 Subject: [PATCH 19/35] Revert "fix typo" This reverts commit 30b5377ab0aa434c051e8ac4cf8d2f3cbe4b1d3c. --- deepmd/tf/lmp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepmd/tf/lmp.py b/deepmd/tf/lmp.py index 1c3c265db2..e378beecfe 100644 --- a/deepmd/tf/lmp.py +++ b/deepmd/tf/lmp.py @@ -80,7 +80,7 @@ def get_library_path(module: str, filename: str) -> List[str]: tf_dir = tf.sysconfig.get_lib() op_dir = str(SHARED_LIB_DIR) -pt_spec = find_spec("torch.lib") +pt_spec = find_spec("torch") if pt_spec is not None: pt_dir = pt_spec.submodule_search_locations[0] else: From a2e4070d48744821e7b73c854240db6c6ef9ac50 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 22:33:56 -0400 Subject: [PATCH 20/35] Revert "set pt dir to LD_LIBRARY_PATH" This reverts commit ced2562d7f15ef3278bb9a13afdd7ab6cab5f69f. --- deepmd/tf/lmp.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/deepmd/tf/lmp.py b/deepmd/tf/lmp.py index e378beecfe..b2e47308ed 100644 --- a/deepmd/tf/lmp.py +++ b/deepmd/tf/lmp.py @@ -6,9 +6,6 @@ from importlib import ( import_module, ) -from importlib.util import ( - find_spec, -) from pathlib import ( Path, ) @@ -80,11 +77,6 @@ def get_library_path(module: str, filename: str) -> List[str]: tf_dir = tf.sysconfig.get_lib() op_dir = str(SHARED_LIB_DIR) -pt_spec = find_spec("torch") -if pt_spec is not None: - pt_dir = pt_spec.submodule_search_locations[0] -else: - pt_dir = None cuda_library_paths = [] @@ -114,7 +106,6 @@ def get_library_path(module: str, filename: str) -> List[str]: [ os.environ.get(lib_env), tf_dir, - pt_dir, os.path.join(tf_dir, "python"), op_dir, ] From 569a9580cd866204ed2ebbdd6565ab7d8b2b2688 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 22:35:38 -0400 Subject: [PATCH 21/35] Revert "fix openmp issue on macos-arm64" This reverts commit 76225f0e74e36b39d87310e1e385d4b334ece355. Signed-off-by: Jinzhe Zeng --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 48437b3c3b..2d0aeecf3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -218,8 +218,6 @@ PIP_PREFER_BINARY = "1" DP_LAMMPS_VERSION = "stable_2Aug2023_update3" DP_ENABLE_IPI = "1" DP_ENABLE_PYTORCH = "1" -# for unclear reason, when enabling PyTorch, OpenMP is found accidentally -CMAKE_ARGS = "-DCMAKE_DISABLE_FIND_PACKAGE_OpenMP=1" [[tool.cibuildwheel.overrides]] # error: 'value' is unavailable: introduced in macOS 10.13 From 6ea694d12750e38a79720f82784eff3219989850 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 22:36:00 -0400 Subject: [PATCH 22/35] Revert "fix build on macos/windows" This reverts commit 4c28b7570b606d6b532e9cc3d99d9b59044fdfde. --- source/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 6a3103616e..3a357e1321 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -219,9 +219,6 @@ if(ENABLE_PYTORCH AND NOT DEEPMD_C_ROOT) set(OP_CXX_ABI ${CMAKE_MATCH_1}) add_definitions(-D_GLIBCXX_USE_CXX11_ABI=${OP_CXX_ABI}) endif() - else() - # Maybe in macos/windows - set(OP_CXX_ABI_PT ${OP_CXX_ABI}) endif() # get torch directory get the directory of the target "torch" get_target_property(_TORCH_LOCATION torch LOCATION) From ed4bcdaf9fb5863dbec1f6be5e6a5a09314a6e73 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 22:38:17 -0400 Subject: [PATCH 23/35] revert macos and windows Signed-off-by: Jinzhe Zeng --- pyproject.toml | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2d0aeecf3d..d0c8cb09ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -208,23 +208,12 @@ manylinux-x86_64-image = "quay.io/pypa/manylinux_2_28_x86_64:2022-11-19-1b19e81" manylinux-aarch64-image = "manylinux_2_28" [tool.cibuildwheel.macos] +environment = { PIP_PREFER_BINARY="1", DP_LAMMPS_VERSION="stable_2Aug2023_update3", DP_ENABLE_IPI="1" } before-all = [ - '''pip install -i https://pypi.anaconda.org/mpi4py/simple mpich''', + """brew install mpich""", ] repair-wheel-command = """delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} --ignore-missing-dependencies""" -[tool.cibuildwheel.macos.environment] -PIP_PREFER_BINARY = "1" -DP_LAMMPS_VERSION = "stable_2Aug2023_update3" -DP_ENABLE_IPI = "1" -DP_ENABLE_PYTORCH = "1" - -[[tool.cibuildwheel.overrides]] -# error: 'value' is unavailable: introduced in macOS 10.13 -select = "*-macosx_x86_64" -inherit.environment = "append" -environment.MACOSX_DEPLOYMENT_TARGET = "10.13" - [tool.cibuildwheel.linux] repair-wheel-command = "auditwheel repair --exclude libtensorflow_framework.so.2 --exclude libtensorflow_framework.so.1 --exclude libtensorflow_framework.so --exclude _pywrap_tensorflow_internal.so --exclude libtensorflow_cc.so.2 --exclude libc10.so --exclude libtorch.so --exclude libtorch_cpu.so -w {dest_dir} {wheel}" environment-pass = [ @@ -262,14 +251,12 @@ UV_EXTRA_INDEX_URL = "https://download.pytorch.org/whl/cpu" CMAKE_PREFIX_PATH="/opt/python/cp311-cp311/" [tool.cibuildwheel.windows] +environment = { PIP_PREFER_BINARY="1" } test-extras = ["cpu"] test-command = [ "python -m deepmd -h", "dp -h", ] -[tool.cibuildwheel.windows.environment] -PIP_PREFER_BINARY = "1" -DP_ENABLE_PYTORCH = "1" # One can run `tox` or `tox -e gpu` # to run pytest in an isolated environment From 96affe03a5f99ecde78d09c250e64781fbf77048 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 23:40:05 -0400 Subject: [PATCH 24/35] base version Signed-off-by: Jinzhe Zeng --- backend/find_pytorch.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/find_pytorch.py b/backend/find_pytorch.py index 82d8f8bf71..32532ba77d 100644 --- a/backend/find_pytorch.py +++ b/backend/find_pytorch.py @@ -24,6 +24,10 @@ Union, ) +from packaging.version import ( + Version, +) + @lru_cache def find_pytorch() -> Tuple[Optional[str], List[str]]: @@ -104,10 +108,11 @@ def get_pt_requirement(pt_version: str = "") -> dict: return {"torch": []} if pt_version == "": pt_version = os.environ.get("PYTORCH_VERSION", "") + base_version = Version(pt_version).base_version return { "torch": [ - f"torch=={pt_version}" if pt_version != "" else "torch>=2a", + f"torch=={base_version}" if pt_version != "" else "torch>=2a", ], } From da610901b50f4a620a712d298fc16c3f57dd1397 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 21 Jun 2024 23:57:50 -0400 Subject: [PATCH 25/35] only parse version when version is not "" Signed-off-by: Jinzhe Zeng --- backend/find_pytorch.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/find_pytorch.py b/backend/find_pytorch.py index 32532ba77d..9be1e7ba3e 100644 --- a/backend/find_pytorch.py +++ b/backend/find_pytorch.py @@ -108,11 +108,12 @@ def get_pt_requirement(pt_version: str = "") -> dict: return {"torch": []} if pt_version == "": pt_version = os.environ.get("PYTORCH_VERSION", "") - base_version = Version(pt_version).base_version return { "torch": [ - f"torch=={base_version}" if pt_version != "" else "torch>=2a", + f"torch=={Version(pt_version).base_version}" + if pt_version != "" + else "torch>=2a", ], } From b5f336d24985662085791f9638f79779c8de8b55 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 22 Jun 2024 01:35:09 -0400 Subject: [PATCH 26/35] Revert "revert macos and windows" This reverts commit ed4bcdaf9fb5863dbec1f6be5e6a5a09314a6e73. --- pyproject.toml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d0c8cb09ba..2d0aeecf3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -208,12 +208,23 @@ manylinux-x86_64-image = "quay.io/pypa/manylinux_2_28_x86_64:2022-11-19-1b19e81" manylinux-aarch64-image = "manylinux_2_28" [tool.cibuildwheel.macos] -environment = { PIP_PREFER_BINARY="1", DP_LAMMPS_VERSION="stable_2Aug2023_update3", DP_ENABLE_IPI="1" } before-all = [ - """brew install mpich""", + '''pip install -i https://pypi.anaconda.org/mpi4py/simple mpich''', ] repair-wheel-command = """delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} --ignore-missing-dependencies""" +[tool.cibuildwheel.macos.environment] +PIP_PREFER_BINARY = "1" +DP_LAMMPS_VERSION = "stable_2Aug2023_update3" +DP_ENABLE_IPI = "1" +DP_ENABLE_PYTORCH = "1" + +[[tool.cibuildwheel.overrides]] +# error: 'value' is unavailable: introduced in macOS 10.13 +select = "*-macosx_x86_64" +inherit.environment = "append" +environment.MACOSX_DEPLOYMENT_TARGET = "10.13" + [tool.cibuildwheel.linux] repair-wheel-command = "auditwheel repair --exclude libtensorflow_framework.so.2 --exclude libtensorflow_framework.so.1 --exclude libtensorflow_framework.so --exclude _pywrap_tensorflow_internal.so --exclude libtensorflow_cc.so.2 --exclude libc10.so --exclude libtorch.so --exclude libtorch_cpu.so -w {dest_dir} {wheel}" environment-pass = [ @@ -251,12 +262,14 @@ UV_EXTRA_INDEX_URL = "https://download.pytorch.org/whl/cpu" CMAKE_PREFIX_PATH="/opt/python/cp311-cp311/" [tool.cibuildwheel.windows] -environment = { PIP_PREFER_BINARY="1" } test-extras = ["cpu"] test-command = [ "python -m deepmd -h", "dp -h", ] +[tool.cibuildwheel.windows.environment] +PIP_PREFER_BINARY = "1" +DP_ENABLE_PYTORCH = "1" # One can run `tox` or `tox -e gpu` # to run pytest in an isolated environment From 6306a41a1c7aee2f983a4b6d6827863632deb87f Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 22 Jun 2024 01:39:06 -0400 Subject: [PATCH 27/35] skip api_cc Signed-off-by: Jinzhe Zeng --- source/api_cc/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/api_cc/CMakeLists.txt b/source/api_cc/CMakeLists.txt index 6a60a91b57..228a6657d3 100644 --- a/source/api_cc/CMakeLists.txt +++ b/source/api_cc/CMakeLists.txt @@ -16,7 +16,10 @@ if(ENABLE_TENSORFLOW) TensorFlow::tensorflow_framework) target_compile_definitions(${libname} PRIVATE BUILD_TENSORFLOW) endif() -if(ENABLE_PYTORCH AND "${OP_CXX_ABI_PT}" EQUAL "${OP_CXX_ABI}") +if(ENABLE_PYTORCH + AND "${OP_CXX_ABI_PT}" EQUAL "${OP_CXX_ABI}" + # LAMMPS and i-PI in the Python package are not ready - needs more work + AND NOT BUILD_PY_IF) target_link_libraries(${libname} PRIVATE "${TORCH_LIBRARIES}") target_compile_definitions(${libname} PRIVATE BUILD_PYTORCH) endif() From 57fdbd46f00796efc31373787bff45482827f349 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 22 Jun 2024 01:41:14 -0400 Subject: [PATCH 28/35] test import Signed-off-by: Jinzhe Zeng --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2d0aeecf3d..04a431208f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -195,11 +195,12 @@ replacement = '\1="https://github.com/deepmodeling/deepmd-kit/raw/master/\g<2>"' [tool.cibuildwheel] test-command = [ "python -m deepmd -h", + """python -c "import deepmd.tf;import deepmd.pt" """, "dp -h", "dp_ipi", "pytest {project}/source/tests/tf/test_lammps.py" ] -test-extras = ["cpu", "test", "lmp", "ipi"] +test-extras = ["cpu", "test", "lmp", "ipi", "torch"] build = ["cp310-*"] skip = ["*-win32", "*-manylinux_i686", "*-musllinux*"] # TODO: uncomment to use the latest image when CUDA 11 is deprecated From b42dd24738f8045ec9a06107265fddbfcf15f396 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 22 Jun 2024 01:42:26 -0400 Subject: [PATCH 29/35] update build Signed-off-by: Jinzhe Zeng --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 04a431208f..40316d4752 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -201,7 +201,7 @@ test-command = [ "pytest {project}/source/tests/tf/test_lammps.py" ] test-extras = ["cpu", "test", "lmp", "ipi", "torch"] -build = ["cp310-*"] +build = ["cp311-*"] skip = ["*-win32", "*-manylinux_i686", "*-musllinux*"] # TODO: uncomment to use the latest image when CUDA 11 is deprecated # manylinux-x86_64-image = "manylinux_2_28" From e0c60595dfcb524cb864540ae98e88fd7a6c54dc Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 22 Jun 2024 01:43:38 -0400 Subject: [PATCH 30/35] update documentation Signed-off-by: Jinzhe Zeng --- doc/install/easy-install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install/easy-install.md b/doc/install/easy-install.md index 0c56fdb0c5..0d012d79b2 100644 --- a/doc/install/easy-install.md +++ b/doc/install/easy-install.md @@ -132,7 +132,7 @@ pip install deepmd-kit[cpu] pip install deepmd-kit[gpu,cu12,torch,lmp,ipi] ``` -MPICH is required for parallel running. (The macOS arm64 package doesn't support MPI yet.) +MPICH is required for parallel running. It is suggested to install the package into an isolated environment. The supported platform includes Linux x86-64 and aarch64 with GNU C Library 2.28 or above, macOS x86-64 and arm64, and Windows x86-64. From 6a88f38e3afdd62d0be09ffd7121c9ed2701e348 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 22 Jun 2024 01:45:57 -0400 Subject: [PATCH 31/35] update documentation Signed-off-by: Jinzhe Zeng --- doc/install/easy-install.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/install/easy-install.md b/doc/install/easy-install.md index 0d012d79b2..8d3ec16e36 100644 --- a/doc/install/easy-install.md +++ b/doc/install/easy-install.md @@ -134,6 +134,10 @@ pip install deepmd-kit[gpu,cu12,torch,lmp,ipi] MPICH is required for parallel running. +:::{Warning} +When installing from pip, only the TensorFlow {{ tensorflow_icon }} backend is supported with LAMMPS and i-PI. +::: + It is suggested to install the package into an isolated environment. The supported platform includes Linux x86-64 and aarch64 with GNU C Library 2.28 or above, macOS x86-64 and arm64, and Windows x86-64. A specific version of TensorFlow and PyTorch which is compatible with DeePMD-kit will be also installed. From 6a9c2946f6882e6f9cfa31f76aa88df575d692bc Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 22 Jun 2024 03:25:46 -0400 Subject: [PATCH 32/35] Revert "Revert "fix openmp issue on macos-arm64"" This reverts commit 569a9580cd866204ed2ebbdd6565ab7d8b2b2688. --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 40316d4752..2b4f8212d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -219,6 +219,8 @@ PIP_PREFER_BINARY = "1" DP_LAMMPS_VERSION = "stable_2Aug2023_update3" DP_ENABLE_IPI = "1" DP_ENABLE_PYTORCH = "1" +# for unclear reason, when enabling PyTorch, OpenMP is found accidentally +CMAKE_ARGS = "-DCMAKE_DISABLE_FIND_PACKAGE_OpenMP=1" [[tool.cibuildwheel.overrides]] # error: 'value' is unavailable: introduced in macOS 10.13 From 4c38fad9e139fa48b72b764ffa2eec9422612ba2 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 22 Jun 2024 03:39:59 -0400 Subject: [PATCH 33/35] version prefix matching Signed-off-by: Jinzhe Zeng --- backend/find_pytorch.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/find_pytorch.py b/backend/find_pytorch.py index 9be1e7ba3e..5a568ecd18 100644 --- a/backend/find_pytorch.py +++ b/backend/find_pytorch.py @@ -111,7 +111,11 @@ def get_pt_requirement(pt_version: str = "") -> dict: return { "torch": [ - f"torch=={Version(pt_version).base_version}" + # uv has different local version behaviors, i.e. `==2.3.1` cannot match `==2.3.1+cpu` + # https://github.com/astral-sh/uv/blob/main/PIP_COMPATIBILITY.md#local-version-identifiers + # luckily, .* (prefix matching) defined in PEP 440 can match any local version + # https://peps.python.org/pep-0440/#version-matching + f"torch=={Version(pt_version).base_version}.*" if pt_version != "" else "torch>=2a", ], From f9e23735a66114856ad1e6213d77d9a28a599839 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 22 Jun 2024 04:46:27 -0400 Subject: [PATCH 34/35] fix the typing of pt_path and tf_path Signed-off-by: Jinzhe Zeng --- backend/find_pytorch.py | 2 +- backend/find_tensorflow.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/find_pytorch.py b/backend/find_pytorch.py index 5a568ecd18..04f297a963 100644 --- a/backend/find_pytorch.py +++ b/backend/find_pytorch.py @@ -123,7 +123,7 @@ def get_pt_requirement(pt_version: str = "") -> dict: @lru_cache -def get_pt_version(pt_path: Union[str, Path]) -> str: +def get_pt_version(pt_path: Optional[Union[str, Path]]) -> str: """Get TF version from a TF Python library path. Parameters diff --git a/backend/find_tensorflow.py b/backend/find_tensorflow.py index 8ba62c9814..06452ab1f5 100644 --- a/backend/find_tensorflow.py +++ b/backend/find_tensorflow.py @@ -204,7 +204,7 @@ def get_tf_requirement(tf_version: str = "") -> dict: @lru_cache -def get_tf_version(tf_path: Union[str, Path]) -> str: +def get_tf_version(tf_path: Optional[Union[str, Path]]) -> str: """Get TF version from a TF Python library path. Parameters From 896e99588c04715e8a343e9622d40c576f438444 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 22 Jun 2024 04:47:11 -0400 Subject: [PATCH 35/35] add torch to windows test-extras Signed-off-by: Jinzhe Zeng --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2b4f8212d9..d9cbeb44e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -265,7 +265,7 @@ UV_EXTRA_INDEX_URL = "https://download.pytorch.org/whl/cpu" CMAKE_PREFIX_PATH="/opt/python/cp311-cp311/" [tool.cibuildwheel.windows] -test-extras = ["cpu"] +test-extras = ["cpu", "torch"] test-command = [ "python -m deepmd -h", "dp -h",