Merge branch 'paddle2' into add_zinc_protein

README.md

@@ -39,7 +39,7 @@
 
 ``` sh
 cd ./source/lib/paddle_src
-python setup_ins.py install
+python custom_op_install.py install
 ```
 
 安装完毕之后建议运行如下命令测试一下 python 端自定义算子在 CPU、GPU 上的正确性：
@@ -91,7 +91,7 @@ dp freeze -i ${WEIGHT_PATH} -o ${DUMP_PATH}
 
 ### 2.5 在 LAMMPS(GPU) 中推理
 
-1. 修改 `examples/water/lmp/in.lammps` 文件，将 `pair_style deepmd` 后面的路径改为 **2.3 导出静态图模型** 这一章节内设置好的 DUMP_PATH 的值
+1. 修改 `examples/water/lmp/in.lammps` 文件，将 `pair_style deepmd` 后面的路径改为 **2.3 导出静态图模型** 这一章节内设置好的 DUMP_PATH 的值(末尾不需要加 `.pdmodel` 或 `.pdiparams`)
 
     ``` suggestion
     pair_style  deepmd "path/to/your_dump"
@@ -118,17 +118,22 @@ dp freeze -i ${WEIGHT_PATH} -o ${DUMP_PATH}
     # 下载并解压 lammps 源码
     wget https://github.com/lammps/lammps/archive/stable_2Aug2023_update1.tar.gz
     tar xf stable_2Aug2023_update1.tar.gz
+
     # LAMMPS_DIR 设置为 LAMMPS 的安装目录
     export LAMMPS_DIR="/path/to/lammps-stable_2Aug2023_update1"
 
     # 设置推理时的 GPU 卡号
     export CUDA_VISIBLE_DEVICES=0
+
     # PADDLE_DIR 设置为第二步 clone下来的 Paddle 目录
     export PADDLE_DIR="/path/to/Paddle"
+
     # DEEPMD_DIR 设置为本项目的根目录
     export DEEPMD_DIR="/path/to/deepmd-kit"
+
     # PADDLE_INFERENCE_DIR 设置为第二步编译得到的 Paddle 推理库目录
     export PADDLE_INFERENCE_DIR="/path/to/paddle_inference_install_dir"
+
     # TENSORFLOW_DIR 设置为 tensorflow 的安装目录，可用 pip show tensorflow 确定
     export TENSORFLOW_DIR="/path/to/tensorflow"
 
@@ -149,7 +154,7 @@ dp freeze -i ${WEIGHT_PATH} -o ${DUMP_PATH}
     export DEEPMD_INSTALL_DIR="path/to/deepmd_root"
 
     # 开始编译
-    cmake -DCMAKE_INSTALL_PREFIX=${DEEPMD_INSTALL_DIR} -DPADDLE_ROOT=${PADDLE_INFERENCE_DIR} \
+    cmake -DCMAKE_INSTALL_PREFIX=${DEEPMD_INSTALL_DIR} \
         -DUSE_CUDA_TOOLKIT=TRUE \
         -DTENSORFLOW_ROOT=${TENSORFLOW_DIR} \
         -DPADDLE_LIB=${PADDLE_INFERENCE_DIR} \
@@ -168,7 +173,6 @@ dp freeze -i ${WEIGHT_PATH} -o ${DUMP_PATH}
     cd ${DEEPMD_DIR}/examples/water/lmp
 
     lmp_serial -in in.lammps
-    ```
 
 4. [可选]直接运行推理
 

backend/dp_backend.py

@@ -1,14 +1,17 @@
+# SPDX-License-Identifier: LGPL-3.0-or-later
 """A PEP-517 backend to find TensorFlow."""
 from typing import (
     List,
 )
 
-from find_tensorflow import (
+from scikit_build_core import build as _orig
+
+from .find_tensorflow import (
     find_tensorflow,
 )
-
-# TODO: switch to scikit_build_core after it is available
-from setuptools import build_meta as _orig
+from .read_env import (
+    set_scikit_build_env,
+)
 
 __all__ = [
     "build_sdist",
@@ -23,10 +26,14 @@ def __dir__() -> List[str]:
     return __all__
 
 
+set_scikit_build_env()
+
 prepare_metadata_for_build_wheel = _orig.prepare_metadata_for_build_wheel
 build_wheel = _orig.build_wheel
 build_sdist = _orig.build_sdist
 get_requires_for_build_sdist = _orig.get_requires_for_build_sdist
+prepare_metadata_for_build_editable = _orig.prepare_metadata_for_build_editable
+build_editable = _orig.build_editable
 
 
 def get_requires_for_build_wheel(
@@ -35,7 +42,7 @@ def get_requires_for_build_wheel(
     return _orig.get_requires_for_build_wheel(config_settings) + find_tensorflow()[1]
 
 
-# TODO: export get_requires_for_build_editable, prepare_metadata_for_build_editable, build_editable
-# after scikit-build is ready
-# See https://github.com/scikit-build/scikit-build/issues/740
-# Now we use the legacy-editable mode
+def get_requires_for_build_editable(
+    config_settings: dict,
+) -> List[str]:
+    return _orig.get_requires_for_build_editable(config_settings) + find_tensorflow()[1]

backend/dynamic_metadata.py

@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: LGPL-3.0-or-later
+from typing import (
+    Dict,
+    List,
+    Optional,
+)
+
+from .find_tensorflow import (
+    get_tf_requirement,
+)
+from .read_env import (
+    get_argument_from_env,
+)
+
+__all__ = ["dynamic_metadata"]
+
+
+def __dir__() -> List[str]:
+    return __all__
+
+
+def dynamic_metadata(
+    field: str,
+    settings: Optional[Dict[str, object]] = None,
+) -> str:
+    assert field in ["optional-dependencies", "entry-points", "scripts"]
+    _, _, find_libpython_requires, extra_scripts, tf_version = get_argument_from_env()
+    if field == "scripts":
+        return {
+            "dp": "deepmd_cli.main:main",
+            **extra_scripts,
+        }
+    elif field == "optional-dependencies":
+        return {
+            "test": [
+                "dpdata>=0.1.9",
+                "ase",
+                "pytest",
+                "pytest-cov",
+                "pytest-sugar",
+                "dpgui",
+            ],
+            "docs": [
+                "sphinx>=3.1.1",
+                "sphinx_rtd_theme>=1.0.0rc1",
+                "sphinx_markdown_tables",
+                "myst-nb",
+                "breathe",
+                "exhale",
+                "numpydoc",
+                "ase",
+                "deepmodeling-sphinx>=0.1.0",
+                "dargs>=0.3.4",
+                "sphinx-argparse",
+                "pygments-lammps",
+                "sphinxcontrib-bibtex",
+            ],
+            "lmp": [
+                "lammps~=2023.8.2.1.0",
+                *find_libpython_requires,
+            ],
+            "ipi": [
+                "i-PI",
+                *find_libpython_requires,
+            ],
+            "gui": [
+                "dpgui",
+            ],
+            **get_tf_requirement(tf_version),
+            "cu11": [
+                "nvidia-cuda-runtime-cu11",
+                "nvidia-cublas-cu11",
+                "nvidia-cufft-cu11",
+                "nvidia-curand-cu11",
+                "nvidia-cusolver-cu11",
+                "nvidia-cusparse-cu11",
+                "nvidia-cudnn-cu11",
+                "nvidia-cuda-nvcc-cu11",
+            ],
+            "cu12": [
+                "nvidia-cuda-runtime-cu12",
+                "nvidia-cublas-cu12",
+                "nvidia-cufft-cu12",
+                "nvidia-curand-cu12",
+                "nvidia-cusolver-cu12",
+                "nvidia-cusparse-cu12",
+                "nvidia-cudnn-cu12",
+                "nvidia-cuda-nvcc-cu12",
+            ],
+        }

backend/read_env.py

@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: LGPL-3.0-or-later
+"""Read environment variables to configure the build."""
+
+import os
+from functools import (
+    lru_cache,
+)
+from typing import (
+    Tuple,
+)
+
+from packaging.version import (
+    Version,
+)
+
+from .find_tensorflow import (
+    find_tensorflow,
+    get_tf_version,
+)
+
+
+@lru_cache()
+def get_argument_from_env() -> Tuple[str, list, list, dict, str]:
+    """Get the arguments from environment variables.
+
+    The environment variables are assumed to be not changed during the build.
+
+    Returns
+    -------
+    str
+        The minimum required CMake version.
+    list of str
+        The CMake arguments.
+    list of str
+        The requirements for the build.
+    dict
+        The extra scripts to be installed.
+    str
+        The TensorFlow version.
+    """
+    cmake_args = []
+    extra_scripts = {}
+    # get variant option from the environment varibles, available: cpu, cuda, rocm
+    dp_variant = os.environ.get("DP_VARIANT", "cpu").lower()
+    if dp_variant == "cpu" or dp_variant == "":
+        cmake_minimum_required_version = "3.16"
+    elif dp_variant == "cuda":
+        cmake_minimum_required_version = "3.23"
+        cmake_args.append("-DUSE_CUDA_TOOLKIT:BOOL=TRUE")
+        cuda_root = os.environ.get("CUDAToolkit_ROOT")
+        if cuda_root:
+            cmake_args.append(f"-DCUDAToolkit_ROOT:STRING={cuda_root}")
+    elif dp_variant == "rocm":
+        cmake_minimum_required_version = "3.21"
+        cmake_args.append("-DUSE_ROCM_TOOLKIT:BOOL=TRUE")
+        rocm_root = os.environ.get("ROCM_ROOT")
+        if rocm_root:
+            cmake_args.append(f"-DCMAKE_HIP_COMPILER_ROCM_ROOT:STRING={rocm_root}")
+        hipcc_flags = os.environ.get("HIP_HIPCC_FLAGS")
+        if hipcc_flags is not None:
+            os.environ["HIPFLAGS"] = os.environ.get("HIPFLAGS", "") + " " + hipcc_flags
+    else:
+        raise RuntimeError("Unsupported DP_VARIANT option: %s" % dp_variant)
+
+    if os.environ.get("DP_BUILD_TESTING", "0") == "1":
+        cmake_args.append("-DBUILD_TESTING:BOOL=TRUE")
+    if os.environ.get("DP_ENABLE_NATIVE_OPTIMIZATION", "0") == "1":
+        cmake_args.append("-DENABLE_NATIVE_OPTIMIZATION:BOOL=TRUE")
+    dp_lammps_version = os.environ.get("DP_LAMMPS_VERSION", "")
+    dp_ipi = os.environ.get("DP_ENABLE_IPI", "0")
+    if dp_lammps_version != "" or dp_ipi == "1":
+        cmake_args.append("-DBUILD_CPP_IF:BOOL=TRUE")
+        cmake_args.append("-DUSE_TF_PYTHON_LIBS:BOOL=TRUE")
+    else:
+        cmake_args.append("-DBUILD_CPP_IF:BOOL=FALSE")
+
+    if dp_lammps_version != "":
+        cmake_args.append(f"-DLAMMPS_VERSION={dp_lammps_version}")
+    if dp_ipi == "1":
+        cmake_args.append("-DENABLE_IPI:BOOL=TRUE")
+        extra_scripts["dp_ipi"] = "deepmd.entrypoints.ipi:dp_ipi"
+
+    tf_install_dir, _ = find_tensorflow()
+    tf_version = get_tf_version(tf_install_dir)
+    if tf_version == "" or Version(tf_version) >= Version("2.12"):
+        find_libpython_requires = []
+    else:
+        find_libpython_requires = ["find_libpython"]
+    cmake_args.append(f"-DTENSORFLOW_VERSION={tf_version}")
+
+    cmake_args = [
+        f"-DTENSORFLOW_ROOT:PATH={tf_install_dir}",
+        "-DBUILD_PY_IF:BOOL=TRUE",
+        *cmake_args,
+    ]
+    return (
+        cmake_minimum_required_version,
+        cmake_args,
+        find_libpython_requires,
+        extra_scripts,
+        tf_version,
+    )
+
+
+def set_scikit_build_env():
+    """Set scikit-build environment variables before executing scikit-build."""
+    cmake_minimum_required_version, cmake_args, _, _, _ = get_argument_from_env()
+    os.environ["SKBUILD_CMAKE_MINIMUM_VERSION"] = cmake_minimum_required_version
+    os.environ["SKBUILD_CMAKE_ARGS"] = ";".join(cmake_args)

data/json/json2yaml.py

@@ -19,7 +19,7 @@ def _main():
     )
 
     # get all json files in dir
-    jsons = [p for p in Path.cwd().glob("*.json")]
+    jsons = list(Path.cwd().glob("*.json"))
     # use the newest as autosuggestion
     jsons.sort(key=lambda x: x.stat().st_mtime, reverse=True)
     jfile = jsons[0]

deepmd/descriptor/se_a.py

@@ -119,7 +119,7 @@ class DescrptSeA(paddle.nn.Layer):
     .. [1] Linfeng Zhang, Jiequn Han, Han Wang, Wissam A. Saidi, Roberto Car, and E. Weinan. 2018.
        End-to-end symmetry preserving inter-atomic potential energy model for finite and extended
        systems. In Proceedings of the 32nd International Conference on Neural Information Processing
-       Systems (NIPS'18). Curran Associates Inc., Red Hook, NY, USA, 4441–4451.
+       Systems (NIPS'18). Curran Associates Inc., Red Hook, NY, USA, 4441-4451.
     """
 
     def __init__(
@@ -893,7 +893,7 @@ def _filter_lower(
         is_exclude=False,
     ):
         """Input env matrix, returns R.G."""
-        outputs_size = [1] + self.filter_neuron
+        outputs_size = [1, *self.filter_neuron]
         # cut-out inputs
         # with natom x (nei_type_i x 4)
         inputs_i = paddle.slice(
@@ -972,6 +972,8 @@ def _filter_lower(
                 transpose_x=True,
             )
 
+    # FIXME: @cast_precision should be annotated when convert to static model
+    # will restore it when it fixed.
     # @cast_precision
     def _filter(
         self,
@@ -1021,14 +1023,16 @@ def _filter(
         nframes = 1
         # natom x (nei x 4)
         shape = inputs.shape
-        outputs_size = [1] + self.filter_neuron
+        outputs_size = [1, *self.filter_neuron]
         outputs_size_2 = self.n_axis_neuron  # 16
         all_excluded = all(
-            [
-                (type_input, type_i) in self.exclude_types  #  set()
+            # FIXME: the bracket '[]' is needed when convert to static model, will be
+            # removed when fixed.
+            [  # noqa
+                (type_input, type_i) in self.exclude_types  #  set() noqa
                 for type_i in range(self.ntypes)
             ]
-        )  # False
+        )
         if all_excluded:
             # all types are excluded so result and qmat should be zeros
             # we can safaly return a zero matrix...

deepmd/descriptor/se_a_ebd.py

@@ -228,7 +228,7 @@ def _embedding_net(
         # natom x (nei x 4)
         inputs = tf.reshape(inputs, [-1, self.ndescrpt])
         shape = inputs.get_shape().as_list()
-        outputs_size = [1] + filter_neuron
+        outputs_size = [1, *filter_neuron]
         with tf.variable_scope(name, reuse=reuse):
             xyz_scatter_total = []
             # with natom x (nei x 4)

deepmd/descriptor/se_a_mask.py

@@ -101,7 +101,7 @@ class DescrptSeAMask(paddle.nn.Layer):
     .. [1] Linfeng Zhang, Jiequn Han, Han Wang, Wissam A. Saidi, Roberto Car, and E. Weinan. 2018.
        End-to-end symmetry preserving inter-atomic potential energy model for finite and extended
        systems. In Proceedings of the 32nd International Conference on Neural Information Processing
-       Systems (NIPS'18). Curran Associates Inc., Red Hook, NY, USA, 4441–4451.
+       Systems (NIPS'18). Curran Associates Inc., Red Hook, NY, USA, 4441-4451.
     """
 
     def __init__(