Merge devel into master (deepmodeling#2871)

.github/labeler.yml

@@ -5,8 +5,8 @@ Python:
 Docs: doc/**/*
 Examples: examples/**/*
 Core: source/lib/**/*
-CUDA: source/lib/src/cuda/**/*
-ROCM: source/lib/src/rocm/**/*
+CUDA: source/lib/src/gpu/**/*
+ROCM: source/lib/src/gpu/**/*
 OP: source/op/**/*
 C++: source/api_cc/**/*
 C: source/api_c/**/*

.github/workflows/build_cc.yml

@@ -20,20 +20,24 @@ jobs:
         - variant: clang
           dp_variant: clang
     steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
         python-version: '3.11'
+        cache: 'pip'
+    - uses: lukka/get-cmake@latest
     - run: python -m pip install tensorflow
-    - run: sudo apt-get update && sudo apt-get install -y nvidia-cuda-toolkit
+    - run: |
+         wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb \
+         && sudo dpkg -i cuda-keyring_1.0-1_all.deb \
+         && sudo apt-get update \
+         && sudo apt-get -y install cuda-cudart-dev-11-8 cuda-nvcc-11-8
       if: matrix.variant == 'cuda'
     - run: |
          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb \
          && sudo dpkg -i cuda-keyring_1.0-1_all.deb \
          && sudo apt-get update \
-         && sudo apt-get -y install cuda-12-0
+         && sudo apt-get -y install cuda-cudart-dev-12-0 cuda-nvcc-12-0
       if: matrix.variant == 'cuda120'
       env:
         DEBIAN_FRONTEND: noninteractive
@@ -44,19 +48,19 @@ jobs:
          && sudo apt-get update \
          && sudo apt-get install -y rocm-dev hipcub-dev
       if: matrix.variant == 'rocm'
-    - run: sudo apt-get update && sudo apt-get install -y clang
-      if: matrix.variant == 'clang'
     - run: source/install/build_cc.sh
       env:
         DP_VARIANT: ${{ matrix.dp_variant }}
         DOWNLOAD_TENSORFLOW: "FALSE"
+        CMAKE_GENERATOR: Ninja
       if: matrix.variant != 'clang'
     - run: source/install/build_cc.sh
       env:
         DP_VARIANT: cpu
         DOWNLOAD_TENSORFLOW: "FALSE"
         CC: clang
         CXX: clang++
+        CMAKE_GENERATOR: Ninja
       if: matrix.variant == 'clang'
     - name: Test files exist
       run: |

.github/workflows/build_wheel.yml

@@ -5,8 +5,24 @@ on:
   pull_request:
 
 jobs:
+  determine-arm64-runner:
+    runs-on: ubuntu-latest
+    permissions: read-all
+    outputs:
+      runner: ${{ steps.set-runner.outputs.runner }}
+    steps:
+      - name: Determine which runner to use for ARM64 build
+        id: set-runner
+        run: |
+          if [ "${{ github.repository_owner }}" == "deepmodeling" ]; then
+            echo "runner=[\"Linux\",\"ARM64\"]" >> $GITHUB_OUTPUT
+          else
+            echo "runner=\"ubuntu-latest\"" >> $GITHUB_OUTPUT
+          fi
+
   build_wheels:
     name: Build wheels for cp${{ matrix.python }}-${{ matrix.platform_id }}
+    needs: determine-arm64-runner
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -33,21 +49,20 @@ jobs:
             platform_id: win_amd64
             dp_variant: cpu
           # linux-aarch64
-          - os: ubuntu-latest
+          - os: ${{ fromJson(needs.determine-arm64-runner.outputs.runner) }}
             python: 310
             platform_id: manylinux_aarch64
             dp_variant: cpu
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
-          submodules: true
           # https://github.com/pypa/setuptools_scm/issues/480
           fetch-depth: 0
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Setup QEMU
-        if: matrix.platform_id == 'manylinux_aarch64'
+        if: matrix.platform_id == 'manylinux_aarch64' && matrix.os == 'ubuntu-latest'
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.15
+        uses: pypa/cibuildwheel@v2.16
         env:
           CIBW_BUILD_VERBOSITY: 1
           CIBW_ARCHS: all
@@ -60,13 +75,12 @@ jobs:
     name: Build source distribution
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         name: Install Python
         with:
           python-version: '3.11'
+          cache: 'pip'
       - run: python -m pip install build
       - name: Build sdist
         run: python -m build --sdist
@@ -94,26 +108,26 @@ jobs:
     needs: [build_wheels]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/download-artifact@v3
         with:
           name: artifact
           path: source/install/docker/dist
       - name: Log in to the Container registry
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175
+        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934
         with:
           images: ghcr.io/deepmodeling/deepmd-kit
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09
         with:
           context: source/install/docker
           push: ${{ github.repository_owner == 'deepmodeling' && github.event_name == 'push' }}

.github/workflows/package_c.yml

@@ -9,7 +9,7 @@ jobs:
     name: Build C library
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Package C library
         run: ./source/install/docker_package_c.sh
       # for download and debug
@@ -30,7 +30,7 @@ jobs:
     needs: [build_c]
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Download artifact
         uses: actions/download-artifact@v3
         with:

.github/workflows/remove_test_cuda_label.yml

@@ -0,0 +1,18 @@
+on:
+  pull_request_target:
+    types:
+      - "labeled"
+name: Test CUDA
+jobs:
+  remove_label:
+    permissions:
+      contents: read
+      pull-requests: write
+    # so one can re-trigger the workflow without manually removing the label
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'deepmodeling' && github.event.label.name == 'Test CUDA'
+    steps:
+    - uses: actions-ecosystem/action-remove-labels@v1
+      with:
+        labels: Test CUDA
+        number: ${{ github.event.pull_request.number }}

.github/workflows/test_cc.yml

@@ -7,21 +7,24 @@ jobs:
     name: Test C++
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
         python-version: '3.11'
+        cache: 'pip'
     - name: Setup MPI
       uses: mpi4py/setup-mpi@v1
       with:
         mpi: mpich
+    - uses: lukka/get-cmake@latest
     - run: python -m pip install tensorflow
     - run: source/install/test_cc_local.sh
       env:
         OMP_NUM_THREADS: 1
         TF_INTRA_OP_PARALLELISM_THREADS: 1
         TF_INTER_OP_PARALLELISM_THREADS: 1
         LMP_CXX11_ABI_0: 1
+        CMAKE_GENERATOR: Ninja
     # test lammps
     # ASE issue: https://gitlab.com/ase/ase/-/merge_requests/2843
     # TODO: remove ase version when ase has new release

.github/workflows/test_cuda.yml

@@ -0,0 +1,60 @@
+on:
+  # manually trigger
+  workflow_dispatch:
+  pull_request:
+    types:
+      - "labeled"
+name: Test CUDA
+jobs:
+  test_cuda:
+    name: Test Python and C++ on CUDA
+    runs-on: nvidia
+    if: github.repository_owner == 'deepmodeling' && github.event.label.name == 'Test CUDA' || github.event_name == 'workflow_dispatch'
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+    - name: Setup MPI
+      uses: mpi4py/setup-mpi@v1
+      with:
+        mpi: mpich
+    - uses: lukka/get-cmake@latest
+    - run: |
+         wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb \
+         && sudo dpkg -i cuda-keyring_1.0-1_all.deb \
+         && sudo apt-get update \
+         && sudo apt-get -y install cuda-11-8 libcudnn8=8.9.5.*-1+cuda11.8
+    - run: python -m pip install -U "pip>=21.3.1,!=23.0.0"
+    - run: pip install -v -e .[gpu,test,lmp,cu11] "ase @ https://github.com/rosswhitfield/ase/archive/edd03571aff6944b77b4a4b055239f3c3e4eeb66.zip"
+      env:
+        DP_BUILD_TESTING: 1
+        DP_VARIANT: cuda
+        CUDA_PATH: /usr/local/cuda-11.8
+    - run: dp --version
+    - run: pytest -s --cov=deepmd --cov=deepmd_cli source/tests --durations=0
+    - run: source/install/test_cc_local.sh
+      env:
+        OMP_NUM_THREADS: 1
+        TF_INTRA_OP_PARALLELISM_THREADS: 1
+        TF_INTER_OP_PARALLELISM_THREADS: 1
+        LMP_CXX11_ABI_0: 1
+        CMAKE_GENERATOR: Ninja
+        DP_VARIANT: cuda
+        DP_USE_MPICH2: 1
+        CUDA_PATH: /usr/local/cuda-11.8
+    - run: |
+        export LD_LIBRARY_PATH=${{ github.workspace }}/dp_test/lib:$CUDA_PATH/lib64:$LD_LIBRARY_PATH
+        export PATH=${{ github.workspace }}/dp_test/bin:$PATH
+        pytest -s --cov=deepmd source/lmp/tests
+        pytest -s --cov=deepmd source/ipi/tests
+      env:
+        OMP_NUM_THREADS: 1
+        TF_INTRA_OP_PARALLELISM_THREADS: 1
+        TF_INTER_OP_PARALLELISM_THREADS: 1
+        LAMMPS_PLUGIN_PATH: ${{ github.workspace }}/dp_test/lib/deepmd_lmp
+        CUDA_PATH: /usr/local/cuda-11.8
+    - uses: codecov/codecov-action@v3
+      with:
+        gcov: true

.github/workflows/test_python.yml

@@ -5,48 +5,43 @@ name: Test Python
 jobs:
   testpython:
     name: Test Python
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         include:
           - python: 3.7
-            gcc: 5
-            tf: 1.14
-          - python: 3.7
-            gcc: 6
-            tf: 1.14
-          - python: 3.7
-            gcc: 7
-            tf: 1.14
-          - python: 3.7
-            gcc: 8
             tf: 1.14
           - python: 3.8
-            gcc: 8
             tf:
           - python: "3.11"
-            gcc: 8
             tf:
 
-    container: ghcr.io/deepmodeling/deepmd-kit-test-environment:py${{ matrix.python }}-gcc${{ matrix.gcc }}-tf${{ matrix.tf }}
     steps:
-    - name: work around permission issue
-      run: git config --global --add safe.directory /__w/deepmd-kit/deepmd-kit
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python }}
+        cache: 'pip'
+    - uses: mpi4py/setup-mpi@v1
+      if: ${{ matrix.tf == '' }}
+      with:
+        mpi: openmpi
     # https://github.com/pypa/pip/issues/11770
     - run: python -m pip install -U "pip>=21.3.1,!=23.0.0"
     - run: pip install -e .[cpu,test]
       env:
-        CC: gcc-${{ matrix.gcc }}
-        CXX: g++-${{ matrix.gcc }}
         TENSORFLOW_VERSION: ${{ matrix.tf }}
         DP_BUILD_TESTING: 1
+    - run: pip install horovod mpi4py
+      if: ${{ matrix.tf == '' }}
+      env:
+        HOROVOD_WITH_TENSORFLOW: 1
+        HOROVOD_WITHOUT_GLOO: 1
     - run: dp --version
     - run: pytest --cov=deepmd --cov=deepmd_cli source/tests --durations=0
     - uses: codecov/codecov-action@v3
       with:
         gcov: true
-        gcov_executable: gcov-${{ matrix.gcc }}
   pass:
     name: Pass testing Python
     needs: [testpython]

.gitignore

@@ -22,7 +22,7 @@ _skbuild
 deepmd_kit.egg-info/
 dist
 .eggs
-_version.py
+/deepmd/_version.py
 venv*
 .vscode/**
 _build