From 6e306fa6eb8fb2841bf31e8a53344f384bb8f577 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 10 Sep 2023 20:04:18 -0400 Subject: [PATCH 1/2] migrate Python build backend to scikit-build-core (#2798) This PR migrates the Python build backend from scikit-build to [scikit-build-core](https://github.com/scikit-build/scikit-build-core), which is being actively developed. There are no breaking changes to users; all tests should be passed without modification. The behavior for editable installation (`pip install -e .`) is changed. Before, the compiled binary libraries (e.g. `libdeepmd_op.sp`) are installed into the source directory. In this PR, the library files are installed into `site-packages` separately, while the Python files are still in the source code and can be modified for development. --------- Signed-off-by: Jinzhe Zeng --- .gitignore | 2 +- MANIFEST.in | 9 - backend/__init__.py | 1 + backend/dp_backend.py | 22 ++- backend/dynamic_metadata.py | 87 +++++++++ backend/find_tensorflow.py | 6 + backend/read_env.py | 109 +++++++++++ deepmd/.gitignore | 2 - deepmd/env.py | 14 +- deepmd/lmp.py | 2 +- {source => deepmd}/op/_add_flt_nvnmd_grad.py | 0 {source => deepmd}/op/_copy_flt_nvnmd_grad.py | 0 .../op/_dotmul_flt_nvnmd_grad.py | 0 {source => deepmd}/op/_flt_nvnmd_grad.py | 0 {source => deepmd}/op/_gelu.py | 0 {source => deepmd}/op/_map_flt_nvnmd_grad.py | 0 .../op/_matmul_fitnet_nvnmd_grad.py | 0 .../op/_matmul_flt2fix_nvnmd.py | 0 .../op/_matmul_flt_nvnmd_grad.py | 0 {source => deepmd}/op/_mul_flt_nvnmd_grad.py | 0 {source => deepmd}/op/_prod_force_grad.py | 0 .../op/_prod_force_se_a_grad.py | 0 .../op/_prod_force_se_a_mask_grad.py | 0 .../op/_prod_force_se_r_grad.py | 0 {source => deepmd}/op/_prod_virial_grad.py | 0 .../op/_prod_virial_se_a_grad.py | 0 .../op/_prod_virial_se_r_grad.py | 0 {source => deepmd}/op/_quantize_nvnmd_grad.py | 0 {source => deepmd}/op/_soft_min_force_grad.py | 0 .../op/_soft_min_virial_grad.py | 0 {source => deepmd}/op/_tabulate_grad.py | 0 .../op/_tanh4_flt_nvnmd_grad.py | 0 deepmd_cli/main.py | 18 +- pyproject.toml | 56 +++++- setup.py | 174 ------------------ source/api_c/CMakeLists.txt | 2 +- source/api_cc/CMakeLists.txt | 2 +- source/config/CMakeLists.txt | 3 +- source/config/__init__.py | 2 + source/ipi/CMakeLists.txt | 10 +- source/lib/CMakeLists.txt | 2 +- source/lib/src/cuda/CMakeLists.txt | 2 +- source/lib/src/cuda/cudart/CMakeLists.txt | 2 +- source/lib/src/rocm/CMakeLists.txt | 2 +- source/lmp/plugin/CMakeLists.txt | 2 +- source/op/CMakeLists.txt | 6 +- 46 files changed, 300 insertions(+), 237 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 backend/__init__.py create mode 100644 backend/dynamic_metadata.py create mode 100644 backend/read_env.py rename {source => deepmd}/op/_add_flt_nvnmd_grad.py (100%) rename {source => deepmd}/op/_copy_flt_nvnmd_grad.py (100%) rename {source => deepmd}/op/_dotmul_flt_nvnmd_grad.py (100%) rename {source => deepmd}/op/_flt_nvnmd_grad.py (100%) rename {source => deepmd}/op/_gelu.py (100%) rename {source => deepmd}/op/_map_flt_nvnmd_grad.py (100%) rename {source => deepmd}/op/_matmul_fitnet_nvnmd_grad.py (100%) rename {source => deepmd}/op/_matmul_flt2fix_nvnmd.py (100%) rename {source => deepmd}/op/_matmul_flt_nvnmd_grad.py (100%) rename {source => deepmd}/op/_mul_flt_nvnmd_grad.py (100%) rename {source => deepmd}/op/_prod_force_grad.py (100%) rename {source => deepmd}/op/_prod_force_se_a_grad.py (100%) rename {source => deepmd}/op/_prod_force_se_a_mask_grad.py (100%) rename {source => deepmd}/op/_prod_force_se_r_grad.py (100%) rename {source => deepmd}/op/_prod_virial_grad.py (100%) rename {source => deepmd}/op/_prod_virial_se_a_grad.py (100%) rename {source => deepmd}/op/_prod_virial_se_r_grad.py (100%) rename {source => deepmd}/op/_quantize_nvnmd_grad.py (100%) rename {source => deepmd}/op/_soft_min_force_grad.py (100%) rename {source => deepmd}/op/_soft_min_virial_grad.py (100%) rename {source => deepmd}/op/_tabulate_grad.py (100%) rename {source => deepmd}/op/_tanh4_flt_nvnmd_grad.py (100%) delete mode 100644 setup.py create mode 100644 source/config/__init__.py diff --git a/.gitignore b/.gitignore index 7401566afd..82d3e4a7da 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,7 @@ _skbuild deepmd_kit.egg-info/ dist .eggs -_version.py +/deepmd/_version.py venv* .vscode/** _build diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 007e125125..0000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,9 +0,0 @@ -prune source/tests -prune source/api_c/tests -prune source/api_cc/tests -prune source/lib/tests -prune source/lmp/tests -prune doc -prune examples -prune data -prune .github diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000000..6ceb116d85 --- /dev/null +++ b/backend/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later diff --git a/backend/dp_backend.py b/backend/dp_backend.py index 9e5932be74..d28afdb239 100644 --- a/backend/dp_backend.py +++ b/backend/dp_backend.py @@ -4,12 +4,14 @@ List, ) -from find_tensorflow import ( +from scikit_build_core import build as _orig + +from .find_tensorflow import ( find_tensorflow, ) - -# TODO: switch to scikit_build_core after it is available -from setuptools import build_meta as _orig +from .read_env import ( + set_scikit_build_env, +) __all__ = [ "build_sdist", @@ -24,10 +26,14 @@ def __dir__() -> List[str]: return __all__ +set_scikit_build_env() + prepare_metadata_for_build_wheel = _orig.prepare_metadata_for_build_wheel build_wheel = _orig.build_wheel build_sdist = _orig.build_sdist get_requires_for_build_sdist = _orig.get_requires_for_build_sdist +prepare_metadata_for_build_editable = _orig.prepare_metadata_for_build_editable +build_editable = _orig.build_editable def get_requires_for_build_wheel( @@ -36,7 +42,7 @@ def get_requires_for_build_wheel( return _orig.get_requires_for_build_wheel(config_settings) + find_tensorflow()[1] -# TODO: export get_requires_for_build_editable, prepare_metadata_for_build_editable, build_editable -# after scikit-build is ready -# See https://github.com/scikit-build/scikit-build/issues/740 -# Now we use the legacy-editable mode +def get_requires_for_build_editable( + config_settings: dict, +) -> List[str]: + return _orig.get_requires_for_build_editable(config_settings) + find_tensorflow()[1] diff --git a/backend/dynamic_metadata.py b/backend/dynamic_metadata.py new file mode 100644 index 0000000000..1270b6031e --- /dev/null +++ b/backend/dynamic_metadata.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +from typing import ( + Dict, + List, + Optional, +) + +from .find_tensorflow import ( + get_tf_requirement, +) +from .read_env import ( + get_argument_from_env, +) + +__all__ = ["dynamic_metadata"] + + +def __dir__() -> List[str]: + return __all__ + + +def dynamic_metadata( + field: str, + settings: Optional[Dict[str, object]] = None, +) -> str: + assert field in ["optional-dependencies", "entry-points", "scripts"] + _, _, find_libpython_requires, extra_scripts, tf_version = get_argument_from_env() + if field == "scripts": + return { + "dp": "deepmd_cli.main:main", + **extra_scripts, + } + elif field == "optional-dependencies": + return { + "test": [ + "dpdata>=0.1.9", + "ase", + "pytest", + "pytest-cov", + "pytest-sugar", + ], + "docs": [ + "sphinx>=3.1.1", + "sphinx_rtd_theme>=1.0.0rc1", + "sphinx_markdown_tables", + "myst-nb", + "breathe", + "exhale", + "numpydoc", + "ase", + "deepmodeling-sphinx>=0.1.0", + "dargs>=0.3.4", + "sphinx-argparse", + "pygments-lammps", + "sphinxcontrib-bibtex", + ], + "lmp": [ + "lammps~=2023.8.2.0.0; platform_system=='Linux'", + "lammps~=2023.8.2.0.0; platform_system!='Linux'", + *find_libpython_requires, + ], + "ipi": [ + "i-PI", + *find_libpython_requires, + ], + **get_tf_requirement(tf_version), + "cu11": [ + "nvidia-cuda-runtime-cu11", + "nvidia-cublas-cu11", + "nvidia-cufft-cu11", + "nvidia-curand-cu11", + "nvidia-cusolver-cu11", + "nvidia-cusparse-cu11", + "nvidia-cudnn-cu11", + "nvidia-cuda-nvcc-cu11", + ], + "cu12": [ + "nvidia-cuda-runtime-cu12", + "nvidia-cublas-cu12", + "nvidia-cufft-cu12", + "nvidia-curand-cu12", + "nvidia-cusolver-cu12", + "nvidia-cusparse-cu12", + "nvidia-cudnn-cu12", + "nvidia-cuda-nvcc-cu12", + ], + } diff --git a/backend/find_tensorflow.py b/backend/find_tensorflow.py index 567d09b75e..8fe3cedb63 100644 --- a/backend/find_tensorflow.py +++ b/backend/find_tensorflow.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: LGPL-3.0-or-later import os import site +from functools import ( + lru_cache, +) from importlib.machinery import ( FileFinder, ) @@ -25,6 +28,7 @@ ) +@lru_cache() def find_tensorflow() -> Tuple[Optional[str], List[str]]: """Find TensorFlow library. @@ -89,6 +93,7 @@ def find_tensorflow() -> Tuple[Optional[str], List[str]]: return tf_install_dir, requires +@lru_cache() def get_tf_requirement(tf_version: str = "") -> dict: """Get TensorFlow requirement (CPU) when TF is not installed. @@ -143,6 +148,7 @@ def get_tf_requirement(tf_version: str = "") -> dict: } +@lru_cache() def get_tf_version(tf_path: Union[str, Path]) -> str: """Get TF version from a TF Python library path. diff --git a/backend/read_env.py b/backend/read_env.py new file mode 100644 index 0000000000..575c1a57de --- /dev/null +++ b/backend/read_env.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +"""Read environment variables to configure the build.""" + +import os +from functools import ( + lru_cache, +) +from typing import ( + Tuple, +) + +from packaging.version import ( + Version, +) + +from .find_tensorflow import ( + find_tensorflow, + get_tf_version, +) + + +@lru_cache() +def get_argument_from_env() -> Tuple[str, list, list, dict, str]: + """Get the arguments from environment variables. + + The environment variables are assumed to be not changed during the build. + + Returns + ------- + str + The minimum required CMake version. + list of str + The CMake arguments. + list of str + The requirements for the build. + dict + The extra scripts to be installed. + str + The TensorFlow version. + """ + cmake_args = [] + extra_scripts = {} + # get variant option from the environment varibles, available: cpu, cuda, rocm + dp_variant = os.environ.get("DP_VARIANT", "cpu").lower() + if dp_variant == "cpu" or dp_variant == "": + cmake_minimum_required_version = "3.16" + elif dp_variant == "cuda": + cmake_minimum_required_version = "3.23" + cmake_args.append("-DUSE_CUDA_TOOLKIT:BOOL=TRUE") + cuda_root = os.environ.get("CUDAToolkit_ROOT") + if cuda_root: + cmake_args.append(f"-DCUDAToolkit_ROOT:STRING={cuda_root}") + elif dp_variant == "rocm": + cmake_minimum_required_version = "3.21" + cmake_args.append("-DUSE_ROCM_TOOLKIT:BOOL=TRUE") + rocm_root = os.environ.get("ROCM_ROOT") + if rocm_root: + cmake_args.append(f"-DCMAKE_HIP_COMPILER_ROCM_ROOT:STRING={rocm_root}") + hipcc_flags = os.environ.get("HIP_HIPCC_FLAGS") + if hipcc_flags: + cmake_args.append(f"-DHIP_HIPCC_FLAGS:STRING={hipcc_flags}") + else: + raise RuntimeError("Unsupported DP_VARIANT option: %s" % dp_variant) + + if os.environ.get("DP_BUILD_TESTING", "0") == "1": + cmake_args.append("-DBUILD_TESTING:BOOL=TRUE") + if os.environ.get("DP_ENABLE_NATIVE_OPTIMIZATION", "0") == "1": + cmake_args.append("-DENABLE_NATIVE_OPTIMIZATION:BOOL=TRUE") + dp_lammps_version = os.environ.get("DP_LAMMPS_VERSION", "") + dp_ipi = os.environ.get("DP_ENABLE_IPI", "0") + if dp_lammps_version != "" or dp_ipi == "1": + cmake_args.append("-DBUILD_CPP_IF:BOOL=TRUE") + cmake_args.append("-DUSE_TF_PYTHON_LIBS:BOOL=TRUE") + else: + cmake_args.append("-DBUILD_CPP_IF:BOOL=FALSE") + + if dp_lammps_version != "": + cmake_args.append(f"-DLAMMPS_VERSION={dp_lammps_version}") + if dp_ipi == "1": + cmake_args.append("-DENABLE_IPI:BOOL=TRUE") + extra_scripts["dp_ipi"] = "deepmd.entrypoints.ipi:dp_ipi" + + tf_install_dir, _ = find_tensorflow() + tf_version = get_tf_version(tf_install_dir) + if tf_version == "" or Version(tf_version) >= Version("2.12"): + find_libpython_requires = [] + else: + find_libpython_requires = ["find_libpython"] + cmake_args.append(f"-DTENSORFLOW_VERSION={tf_version}") + + cmake_args = [ + f"-DTENSORFLOW_ROOT:PATH={tf_install_dir}", + "-DBUILD_PY_IF:BOOL=TRUE", + *cmake_args, + ] + return ( + cmake_minimum_required_version, + cmake_args, + find_libpython_requires, + extra_scripts, + tf_version, + ) + + +def set_scikit_build_env(): + """Set scikit-build environment variables before executing scikit-build.""" + cmake_minimum_required_version, cmake_args, _, _, _ = get_argument_from_env() + os.environ["SKBUILD_CMAKE_MINIMUM_VERSION"] = cmake_minimum_required_version + os.environ["SKBUILD_CMAKE_ARGS"] = ";".join(cmake_args) diff --git a/deepmd/.gitignore b/deepmd/.gitignore index b2b9057ea2..b2d7614637 100644 --- a/deepmd/.gitignore +++ b/deepmd/.gitignore @@ -1,4 +1,2 @@ -op/_*.py pkg_config run_config.ini -!op/__init__.py diff --git a/deepmd/env.py b/deepmd/env.py index 615e89f3ac..d8875cabd2 100644 --- a/deepmd/env.py +++ b/deepmd/env.py @@ -27,6 +27,8 @@ Version, ) +import deepmd.lib + if TYPE_CHECKING: from types import ( ModuleType, @@ -101,7 +103,9 @@ def dlopen_library(module: str, filename: str): "TF_VERSION", ] -SHARED_LIB_MODULE = "op" +SHARED_LIB_MODULE = "lib" +SHARED_LIB_DIR = Path(deepmd.lib.__path__[0]) +CONFIG_FILE = SHARED_LIB_DIR / "run_config.ini" # Python library version try: @@ -361,11 +365,7 @@ def get_module(module_name: str) -> "ModuleType": ext = ".so" prefix = "lib" - module_file = ( - (Path(__file__).parent / SHARED_LIB_MODULE / (prefix + module_name)) - .with_suffix(ext) - .resolve() - ) + module_file = (SHARED_LIB_DIR / (prefix + module_name)).with_suffix(ext).resolve() if not module_file.is_file(): raise FileNotFoundError(f"module {module_name} does not exist") @@ -433,7 +433,7 @@ def get_module(module_name: str) -> "ModuleType": def _get_package_constants( - config_file: Path = Path(__file__).parent / "run_config.ini", + config_file: Path = CONFIG_FILE, ) -> Dict[str, str]: """Read package constants set at compile time by CMake to dictionary. diff --git a/deepmd/lmp.py b/deepmd/lmp.py index e5a3d4904f..a955844758 100644 --- a/deepmd/lmp.py +++ b/deepmd/lmp.py @@ -64,7 +64,7 @@ def get_library_path(module: str) -> List[str]: raise RuntimeError("Unsupported platform") tf_dir = tf.sysconfig.get_lib() -op_dir = str((Path(__file__).parent / "op").absolute()) +op_dir = str((Path(__file__).parent / "lib").absolute()) cuda_library_paths = [] diff --git a/source/op/_add_flt_nvnmd_grad.py b/deepmd/op/_add_flt_nvnmd_grad.py similarity index 100% rename from source/op/_add_flt_nvnmd_grad.py rename to deepmd/op/_add_flt_nvnmd_grad.py diff --git a/source/op/_copy_flt_nvnmd_grad.py b/deepmd/op/_copy_flt_nvnmd_grad.py similarity index 100% rename from source/op/_copy_flt_nvnmd_grad.py rename to deepmd/op/_copy_flt_nvnmd_grad.py diff --git a/source/op/_dotmul_flt_nvnmd_grad.py b/deepmd/op/_dotmul_flt_nvnmd_grad.py similarity index 100% rename from source/op/_dotmul_flt_nvnmd_grad.py rename to deepmd/op/_dotmul_flt_nvnmd_grad.py diff --git a/source/op/_flt_nvnmd_grad.py b/deepmd/op/_flt_nvnmd_grad.py similarity index 100% rename from source/op/_flt_nvnmd_grad.py rename to deepmd/op/_flt_nvnmd_grad.py diff --git a/source/op/_gelu.py b/deepmd/op/_gelu.py similarity index 100% rename from source/op/_gelu.py rename to deepmd/op/_gelu.py diff --git a/source/op/_map_flt_nvnmd_grad.py b/deepmd/op/_map_flt_nvnmd_grad.py similarity index 100% rename from source/op/_map_flt_nvnmd_grad.py rename to deepmd/op/_map_flt_nvnmd_grad.py diff --git a/source/op/_matmul_fitnet_nvnmd_grad.py b/deepmd/op/_matmul_fitnet_nvnmd_grad.py similarity index 100% rename from source/op/_matmul_fitnet_nvnmd_grad.py rename to deepmd/op/_matmul_fitnet_nvnmd_grad.py diff --git a/source/op/_matmul_flt2fix_nvnmd.py b/deepmd/op/_matmul_flt2fix_nvnmd.py similarity index 100% rename from source/op/_matmul_flt2fix_nvnmd.py rename to deepmd/op/_matmul_flt2fix_nvnmd.py diff --git a/source/op/_matmul_flt_nvnmd_grad.py b/deepmd/op/_matmul_flt_nvnmd_grad.py similarity index 100% rename from source/op/_matmul_flt_nvnmd_grad.py rename to deepmd/op/_matmul_flt_nvnmd_grad.py diff --git a/source/op/_mul_flt_nvnmd_grad.py b/deepmd/op/_mul_flt_nvnmd_grad.py similarity index 100% rename from source/op/_mul_flt_nvnmd_grad.py rename to deepmd/op/_mul_flt_nvnmd_grad.py diff --git a/source/op/_prod_force_grad.py b/deepmd/op/_prod_force_grad.py similarity index 100% rename from source/op/_prod_force_grad.py rename to deepmd/op/_prod_force_grad.py diff --git a/source/op/_prod_force_se_a_grad.py b/deepmd/op/_prod_force_se_a_grad.py similarity index 100% rename from source/op/_prod_force_se_a_grad.py rename to deepmd/op/_prod_force_se_a_grad.py diff --git a/source/op/_prod_force_se_a_mask_grad.py b/deepmd/op/_prod_force_se_a_mask_grad.py similarity index 100% rename from source/op/_prod_force_se_a_mask_grad.py rename to deepmd/op/_prod_force_se_a_mask_grad.py diff --git a/source/op/_prod_force_se_r_grad.py b/deepmd/op/_prod_force_se_r_grad.py similarity index 100% rename from source/op/_prod_force_se_r_grad.py rename to deepmd/op/_prod_force_se_r_grad.py diff --git a/source/op/_prod_virial_grad.py b/deepmd/op/_prod_virial_grad.py similarity index 100% rename from source/op/_prod_virial_grad.py rename to deepmd/op/_prod_virial_grad.py diff --git a/source/op/_prod_virial_se_a_grad.py b/deepmd/op/_prod_virial_se_a_grad.py similarity index 100% rename from source/op/_prod_virial_se_a_grad.py rename to deepmd/op/_prod_virial_se_a_grad.py diff --git a/source/op/_prod_virial_se_r_grad.py b/deepmd/op/_prod_virial_se_r_grad.py similarity index 100% rename from source/op/_prod_virial_se_r_grad.py rename to deepmd/op/_prod_virial_se_r_grad.py diff --git a/source/op/_quantize_nvnmd_grad.py b/deepmd/op/_quantize_nvnmd_grad.py similarity index 100% rename from source/op/_quantize_nvnmd_grad.py rename to deepmd/op/_quantize_nvnmd_grad.py diff --git a/source/op/_soft_min_force_grad.py b/deepmd/op/_soft_min_force_grad.py similarity index 100% rename from source/op/_soft_min_force_grad.py rename to deepmd/op/_soft_min_force_grad.py diff --git a/source/op/_soft_min_virial_grad.py b/deepmd/op/_soft_min_virial_grad.py similarity index 100% rename from source/op/_soft_min_virial_grad.py rename to deepmd/op/_soft_min_virial_grad.py diff --git a/source/op/_tabulate_grad.py b/deepmd/op/_tabulate_grad.py similarity index 100% rename from source/op/_tabulate_grad.py rename to deepmd/op/_tabulate_grad.py diff --git a/source/op/_tanh4_flt_nvnmd_grad.py b/deepmd/op/_tanh4_flt_nvnmd_grad.py similarity index 100% rename from source/op/_tanh4_flt_nvnmd_grad.py rename to deepmd/op/_tanh4_flt_nvnmd_grad.py diff --git a/deepmd_cli/main.py b/deepmd_cli/main.py index e3213d8b00..f707bf7589 100644 --- a/deepmd_cli/main.py +++ b/deepmd_cli/main.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: LGPL-3.0-or-later import argparse -import imp +import importlib.util import logging +import os +import sys import textwrap from typing import ( List, @@ -12,11 +14,15 @@ def load_child_module(name): """Load a child module without loading its parent module.""" names = name.split(".") - path = None - for name in names: - f, path, info = imp.find_module(name, path) - path = [path] - return imp.load_module(name, f, path[0], info) + parent_spec = importlib.util.find_spec(names[0]) + paths = os.path.join(*names[1:]) + ".py" + spec = importlib.util.spec_from_file_location( + name, os.path.join(parent_spec.submodule_search_locations[0], paths) + ) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module __version__ = load_child_module("deepmd._version").__version__ diff --git a/pyproject.toml b/pyproject.toml index 687e0284cc..7b8f55d562 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,16 @@ [build-system] requires = [ - "setuptools>=61", - "setuptools_scm[toml]>=6.2", - "wheel", - "scikit-build", - "cmake", - # see https://github.com/scikit-build/scikit-build/releases/tag/0.13.1 - "ninja; platform_system!='Windows'", + # dynamic metadata API is still unstable + # TODO: unpin the upper bound when it is stable + "scikit-build-core>=0.5,<0.6", + "packaging", ] -build-backend = "dp_backend" -backend-path = ["backend"] +build-backend = "backend.dp_backend" +backend-path = ["."] [project] name = "deepmd-kit" -dynamic = ["version", "optional-dependencies", "entry-points"] +dynamic = ["version", "optional-dependencies", "scripts"] description = "A deep learning package for many-body potential energy representation and molecular dynamics" authors = [ {name = "DeepModeling"}, @@ -53,6 +50,9 @@ requires-python = ">=3.7" readme = "README.md" keywords = ["deepmd"] +[project.entry-points."lammps.plugins"] +deepmd = "deepmd.lmp:get_op_dir" + [project.urls] Homepage = "https://github.com/deepmodeling/deepmd-kit" documentation = "https://docs.deepmodeling.com/projects/deepmd" @@ -61,6 +61,42 @@ repository = "https://github.com/deepmodeling/deepmd-kit" [tool.setuptools_scm] write_to = "deepmd/_version.py" +[tool.scikit-build] +experimental = true +minimum-version = "0.5" +cmake.source-dir = "source" +sdist.include = [ + "/deepmd/_version.py", +] +sdist.exclude = [ + "/source/tests", + "/source/api_c/tests", + "/source/api_cc/tests", + "/source/lib/tests", + "/source/lmp/tests", + "/doc", + "/examples", + "/data", + "/.github", +] +wheel.packages = [ + "deepmd", + "deepmd_cli", +] +wheel.py-api = "py37" +build-dir = "build/{wheel_tag}" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.setuptools_scm" + +[tool.scikit-build.metadata.optional-dependencies] +provider = "backend.dynamic_metadata" +provider-path = "backend" + +[tool.scikit-build.metadata.scripts] +provider = "backend.dynamic_metadata" +provider-path = "backend" + [tool.cibuildwheel] test-command = [ "python -m deepmd -h", diff --git a/setup.py b/setup.py deleted file mode 100644 index 7b69c14c40..0000000000 --- a/setup.py +++ /dev/null @@ -1,174 +0,0 @@ -# SPDX-License-Identifier: LGPL-3.0-or-later -"""Setup script for DeePMD-kit package.""" - -import os -import sys - -from packaging.version import ( - Version, -) -from skbuild import ( - setup, -) -from wheel.bdist_wheel import ( - bdist_wheel, -) - -topdir = os.path.abspath(os.path.dirname(__file__)) -sys.path.insert(0, os.path.join(topdir, "backend")) - -from find_tensorflow import ( - find_tensorflow, - get_tf_requirement, - get_tf_version, -) - -cmake_args = [] -extra_scripts = [] -# get variant option from the environment varibles, available: cpu, cuda, rocm -dp_variant = os.environ.get("DP_VARIANT", "cpu").lower() -if dp_variant == "cpu" or dp_variant == "": - cmake_minimum_required_version = "3.16" -elif dp_variant == "cuda": - cmake_minimum_required_version = "3.23" - cmake_args.append("-DUSE_CUDA_TOOLKIT:BOOL=TRUE") - cuda_root = os.environ.get("CUDAToolkit_ROOT") - if cuda_root: - cmake_args.append(f"-DCUDAToolkit_ROOT:STRING={cuda_root}") -elif dp_variant == "rocm": - cmake_minimum_required_version = "3.21" - cmake_args.append("-DUSE_ROCM_TOOLKIT:BOOL=TRUE") - rocm_root = os.environ.get("ROCM_ROOT") - if rocm_root: - cmake_args.append(f"-DCMAKE_HIP_COMPILER_ROCM_ROOT:STRING={rocm_root}") - hipcc_flags = os.environ.get("HIP_HIPCC_FLAGS") - if hipcc_flags: - cmake_args.append(f"-DHIP_HIPCC_FLAGS:STRING={hipcc_flags}") -else: - raise RuntimeError("Unsupported DP_VARIANT option: %s" % dp_variant) - -if os.environ.get("DP_BUILD_TESTING", "0") == "1": - cmake_args.append("-DBUILD_TESTING:BOOL=TRUE") -if os.environ.get("DP_ENABLE_NATIVE_OPTIMIZATION", "0") == "1": - cmake_args.append("-DENABLE_NATIVE_OPTIMIZATION:BOOL=TRUE") -dp_lammps_version = os.environ.get("DP_LAMMPS_VERSION", "") -dp_ipi = os.environ.get("DP_ENABLE_IPI", "0") -if dp_lammps_version != "" or dp_ipi == "1": - cmake_args.append("-DBUILD_CPP_IF:BOOL=TRUE") - cmake_args.append("-DUSE_TF_PYTHON_LIBS:BOOL=TRUE") -else: - cmake_args.append("-DBUILD_CPP_IF:BOOL=FALSE") - -if dp_lammps_version != "": - cmake_args.append(f"-DLAMMPS_VERSION={dp_lammps_version}") -if dp_ipi == "1": - cmake_args.append("-DENABLE_IPI:BOOL=TRUE") - extra_scripts.append("dp_ipi = deepmd.entrypoints.ipi:dp_ipi") - - -tf_install_dir, _ = find_tensorflow() -tf_version = get_tf_version(tf_install_dir) -if tf_version == "" or Version(tf_version) >= Version("2.12"): - find_libpython_requires = [] -else: - find_libpython_requires = ["find_libpython"] -cmake_args.append(f"-DTENSORFLOW_VERSION={tf_version}") - - -class bdist_wheel_abi3(bdist_wheel): - def get_tag(self): - python, abi, plat = super().get_tag() - if python.startswith("cp"): - if tf_version == "" or Version(tf_version) >= Version("2.12"): - return "py38", "none", plat - return "py37", "none", plat - return python, abi, plat - - -# TODO: migrate packages and entry_points to pyproject.toml after scikit-build supports it -# See also https://scikit-build.readthedocs.io/en/latest/usage.html#setuptools-options -setup( - packages=[ - "deepmd", - "deepmd/descriptor", - "deepmd/fit", - "deepmd/infer", - "deepmd/loss", - "deepmd/utils", - "deepmd/loggers", - "deepmd/cluster", - "deepmd/entrypoints", - "deepmd/op", - "deepmd/model", - "deepmd/train", - "deepmd/nvnmd", - "deepmd/nvnmd/data", - "deepmd/nvnmd/descriptor", - "deepmd/nvnmd/entrypoints", - "deepmd/nvnmd/fit", - "deepmd/nvnmd/utils", - "deepmd_cli", - ], - cmake_args=[ - f"-DTENSORFLOW_ROOT:PATH={tf_install_dir}", - "-DBUILD_PY_IF:BOOL=TRUE", - *cmake_args, - ], - cmake_source_dir="source", - cmake_minimum_required_version=cmake_minimum_required_version, - extras_require={ - "test": ["dpdata>=0.1.9", "ase", "pytest", "pytest-cov", "pytest-sugar"], - "docs": [ - "sphinx>=3.1.1", - "sphinx_rtd_theme>=1.0.0rc1", - "sphinx_markdown_tables", - "myst-nb", - "breathe", - "exhale", - "numpydoc", - "ase", - "deepmodeling-sphinx>=0.1.0", - "dargs>=0.3.4", - "sphinx-argparse", - "pygments-lammps", - "sphinxcontrib-bibtex", - ], - "lmp": [ - "lammps~=2023.8.2.0.0; platform_system=='Linux'", - "lammps~=2023.8.2.0.0; platform_system!='Linux'", - *find_libpython_requires, - ], - "ipi": [ - "i-PI", - *find_libpython_requires, - ], - **get_tf_requirement(tf_version), - "cu11": [ - "nvidia-cuda-runtime-cu11", - "nvidia-cublas-cu11", - "nvidia-cufft-cu11", - "nvidia-curand-cu11", - "nvidia-cusolver-cu11", - "nvidia-cusparse-cu11", - "nvidia-cudnn-cu11", - "nvidia-cuda-nvcc-cu11", - ], - "cu12": [ - "nvidia-cuda-runtime-cu12", - "nvidia-cublas-cu12", - "nvidia-cufft-cu12", - "nvidia-curand-cu12", - "nvidia-cusolver-cu12", - "nvidia-cusparse-cu12", - "nvidia-cudnn-cu12", - "nvidia-cuda-nvcc-cu12", - ], - }, - entry_points={ - "console_scripts": ["dp = deepmd_cli.main:main", *extra_scripts], - "lammps.plugins": ["deepmd = deepmd.lmp:get_op_dir"], - }, - cmdclass={ - "bdist_wheel": bdist_wheel_abi3, - }, -) diff --git a/source/api_c/CMakeLists.txt b/source/api_c/CMakeLists.txt index 93b170f59c..f6e741105a 100644 --- a/source/api_c/CMakeLists.txt +++ b/source/api_c/CMakeLists.txt @@ -19,7 +19,7 @@ if(CMAKE_TESTING_ENABLED) endif() if(BUILD_PY_IF) - install(TARGETS ${libname} DESTINATION deepmd/op/) + install(TARGETS ${libname} DESTINATION deepmd/lib/) else(BUILD_PY_IF) install( TARGETS ${libname} diff --git a/source/api_cc/CMakeLists.txt b/source/api_cc/CMakeLists.txt index 7dc836a873..bdcb51a498 100644 --- a/source/api_cc/CMakeLists.txt +++ b/source/api_cc/CMakeLists.txt @@ -33,7 +33,7 @@ endif() target_compile_features(${libname} PUBLIC cxx_std_11) if(BUILD_PY_IF) - install(TARGETS ${libname} DESTINATION deepmd/op/) + install(TARGETS ${libname} DESTINATION deepmd/lib/) else(BUILD_PY_IF) install( TARGETS ${libname} diff --git a/source/config/CMakeLists.txt b/source/config/CMakeLists.txt index eb0bbc8bf4..5473b91f29 100644 --- a/source/config/CMakeLists.txt +++ b/source/config/CMakeLists.txt @@ -3,4 +3,5 @@ configure_file("run_config.ini" "${CMAKE_CURRENT_BINARY_DIR}/run_config.ini" @ONLY) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/run_config.ini DESTINATION deepmd) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/run_config.ini __init__.py + DESTINATION deepmd/lib) diff --git a/source/config/__init__.py b/source/config/__init__.py new file mode 100644 index 0000000000..d8ba46b41f --- /dev/null +++ b/source/config/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +# empty file for find the module diff --git a/source/ipi/CMakeLists.txt b/source/ipi/CMakeLists.txt index adf69f723f..158f98aea5 100644 --- a/source/ipi/CMakeLists.txt +++ b/source/ipi/CMakeLists.txt @@ -24,11 +24,7 @@ endif() target_link_libraries(${ipiname} PRIVATE ${libipiname}) target_include_directories(${ipiname} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../3rdparty/) -if(BUILD_PY_IF) - set(LIB_DIR op) -else(BUILD_PY_IF) - set(LIB_DIR lib) -endif(BUILD_PY_IF) +set(LIB_DIR lib) if(BUILD_PY_IF AND TENSORFLOW_LINK_LIBPYTHON) # ignore undefined reference for libpython @@ -63,8 +59,8 @@ if(CMAKE_TESTING_ENABLED) endif() if(BUILD_PY_IF) - install(TARGETS ${libipiname} DESTINATION deepmd/op/) - install(TARGETS ${ipiname} DESTINATION deepmd/op/) + install(TARGETS ${libipiname} DESTINATION deepmd/lib/) + install(TARGETS ${ipiname} DESTINATION deepmd/lib/) else(BUILD_PY_IF) install(TARGETS ${libipiname} DESTINATION lib/) install(TARGETS ${ipiname} DESTINATION bin/) diff --git a/source/lib/CMakeLists.txt b/source/lib/CMakeLists.txt index bd6b585644..5f5528de3e 100644 --- a/source/lib/CMakeLists.txt +++ b/source/lib/CMakeLists.txt @@ -38,7 +38,7 @@ if(CMAKE_TESTING_ENABLED) endif() if(BUILD_PY_IF) - install(TARGETS ${libname} DESTINATION deepmd/op/) + install(TARGETS ${libname} DESTINATION deepmd/lib/) else(BUILD_PY_IF) install( TARGETS ${libname} diff --git a/source/lib/src/cuda/CMakeLists.txt b/source/lib/src/cuda/CMakeLists.txt index bfdf9cd466..1d5ae690e1 100644 --- a/source/lib/src/cuda/CMakeLists.txt +++ b/source/lib/src/cuda/CMakeLists.txt @@ -56,5 +56,5 @@ if(BUILD_CPP_IF AND NOT BUILD_PY_IF) DESTINATION lib/) endif(BUILD_CPP_IF AND NOT BUILD_PY_IF) if(BUILD_PY_IF) - install(TARGETS deepmd_op_cuda DESTINATION deepmd/op/) + install(TARGETS deepmd_op_cuda DESTINATION deepmd/lib/) endif(BUILD_PY_IF) diff --git a/source/lib/src/cuda/cudart/CMakeLists.txt b/source/lib/src/cuda/cudart/CMakeLists.txt index 7562c2ea1b..e612ad63ed 100644 --- a/source/lib/src/cuda/cudart/CMakeLists.txt +++ b/source/lib/src/cuda/cudart/CMakeLists.txt @@ -9,5 +9,5 @@ if(BUILD_CPP_IF AND NOT BUILD_PY_IF) DESTINATION lib/) endif(BUILD_CPP_IF AND NOT BUILD_PY_IF) if(BUILD_PY_IF) - install(TARGETS deepmd_dyn_cudart DESTINATION deepmd/op/) + install(TARGETS deepmd_dyn_cudart DESTINATION deepmd/lib/) endif(BUILD_PY_IF) diff --git a/source/lib/src/rocm/CMakeLists.txt b/source/lib/src/rocm/CMakeLists.txt index f659973897..1b093977b6 100644 --- a/source/lib/src/rocm/CMakeLists.txt +++ b/source/lib/src/rocm/CMakeLists.txt @@ -35,5 +35,5 @@ if(BUILD_CPP_IF) DESTINATION lib/) endif(BUILD_CPP_IF) if(BUILD_PY_IF) - install(TARGETS deepmd_op_rocm DESTINATION deepmd/op/) + install(TARGETS deepmd_op_rocm DESTINATION deepmd/lib/) endif(BUILD_PY_IF) diff --git a/source/lmp/plugin/CMakeLists.txt b/source/lmp/plugin/CMakeLists.txt index ce879764ee..86b99fe7b5 100644 --- a/source/lmp/plugin/CMakeLists.txt +++ b/source/lmp/plugin/CMakeLists.txt @@ -100,7 +100,7 @@ if(DEFINED LAMMPS_SOURCE_ROOT OR DEFINED LAMMPS_VERSION) endif() if(BUILD_PY_IF) - install(TARGETS ${libname} DESTINATION deepmd/op/) + install(TARGETS ${libname} DESTINATION deepmd/lib/) else(BUILD_PY_IF) install(TARGETS ${libname} DESTINATION lib/) diff --git a/source/op/CMakeLists.txt b/source/op/CMakeLists.txt index 167c6c5396..7a92e259e0 100644 --- a/source/op/CMakeLists.txt +++ b/source/op/CMakeLists.txt @@ -50,7 +50,6 @@ file( prod_virial_grad_multi_device.cc soft_min_force_grad.cc soft_min_virial_grad.cc) -file(GLOB OP_PY *.py) file(GLOB OP_REMAPPER_SRC optimizer/parallel.cc) add_library(${LIB_DEEPMD_OP} MODULE ${OP_SRC} ${OP_REMAPPER_SRC}) @@ -91,9 +90,8 @@ if(BUILD_PY_IF) endif(BUILD_PY_IF) if(BUILD_PY_IF) - install(TARGETS ${LIB_DEEPMD_OP} DESTINATION deepmd/op/) - install(TARGETS op_grads DESTINATION deepmd/op/) - install(FILES ${OP_PY} DESTINATION deepmd/op/) + install(TARGETS ${LIB_DEEPMD_OP} DESTINATION deepmd/lib/) + install(TARGETS op_grads DESTINATION deepmd/lib/) else(BUILD_PY_IF) install(TARGETS ${LIB_DEEPMD_OP} DESTINATION lib/) endif(BUILD_PY_IF) From e1c9173a194d0c93b9b59be891fba1b0a28e374b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yifan=20Li=E6=9D=8E=E4=B8=80=E5=B8=86?= Date: Mon, 11 Sep 2023 16:24:00 -0400 Subject: [PATCH 2/2] fix the error in dist_unit_cvt_factor --- source/lmp/compute_deeptensor_atom.cpp | 2 +- source/lmp/pair_deepmd.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/lmp/compute_deeptensor_atom.cpp b/source/lmp/compute_deeptensor_atom.cpp index 3458ec3c27..88377f5c18 100644 --- a/source/lmp/compute_deeptensor_atom.cpp +++ b/source/lmp/compute_deeptensor_atom.cpp @@ -58,7 +58,7 @@ ComputeDeeptensorAtom::ComputeDeeptensorAtom(LAMMPS *lmp, int narg, char **arg) nmax = 0; - dist_unit_cvt_factor = 1.0 / force->angstrom; + dist_unit_cvt_factor = force->angstrom; } /* ---------------------------------------------------------------------- */ diff --git a/source/lmp/pair_deepmd.cpp b/source/lmp/pair_deepmd.cpp index 9c86dd2622..c49ef51cba 100644 --- a/source/lmp/pair_deepmd.cpp +++ b/source/lmp/pair_deepmd.cpp @@ -351,7 +351,7 @@ PairDeepMD::PairDeepMD(LAMMPS *lmp) "\"units metal\" or \"units real\""); } ener_unit_cvt_factor = force->boltz / 8.617343e-5; - dist_unit_cvt_factor = 1.0 / force->angstrom; + dist_unit_cvt_factor = force->angstrom; force_unit_cvt_factor = ener_unit_cvt_factor / dist_unit_cvt_factor; restartinfo = 1;