Merge pull request #61 from streeve/cuda_build_ci

Add cuda build CI
ORNL · Oct 16, 2023 · 3c13390 · 3c13390
2 parents 71af08c + 14005e1
commit 3c13390
Show file tree

Hide file tree

Showing 3 changed files with 184 additions and 108 deletions.
diff --git a/.github/workflows/CI-e4s-build.yml b/.github/workflows/CI-e4s-build.yml
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -45,7 +45,7 @@ jobs:
     container: ghcr.io/ecp-copa/ci-containers/${{ matrix.distro }}
     steps:
       - name: Checkout kokkos
-        uses: actions/checkout@v2.2.0
+        uses: actions/checkout@v3
         with:
           repository: kokkos/kokkos
           ref: ${{ matrix.kokkos_ver }}
@@ -62,7 +62,7 @@ jobs:
           cmake --build build --parallel 2
           cmake --install build
       - name: Checkout Cabana
-        uses: actions/checkout@v2.2.0
+        uses: actions/checkout@v3
         with:
           repository: ECP-CoPA/Cabana
           # This version is post-release 0.5
@@ -89,7 +89,7 @@ jobs:
           cmake --build build --parallel 2
           cmake --install build
       - name: Checkout CabanaPD
-        uses: actions/checkout@v2.2.0
+        uses: actions/checkout@v3
       - name: Build CabanaPD
         run: |
           cmake -B build \
@@ -106,3 +106,172 @@ jobs:
       - name: Test CabanaPD
         run: |
           CTEST_OUTPUT_ON_FAILURE=1 cmake --build build --target test
+  HIP:
+    defaults:
+      run:
+        shell: bash
+    env:
+      # Needed because FindMPI relies heavily on the compiler wrappers, which aren't used with hipcc
+      MPI_LOCATION: /usr/lib/x86_64-linux-gnu/openmpi/
+    strategy:
+      matrix:
+        cxx: ['hipcc']
+        cmake_build_type: ['Release']
+        kokkos_ver: ['3.6.01']
+    runs-on: ubuntu-20.04
+    container: ghcr.io/ecp-copa/ci-containers/rocm:latest
+    steps:
+      - name: Checkout json
+        uses: actions/checkout@v3
+        with:
+          repository: nlohmann/json
+          ref: v3.11.2
+          path: json
+      - name: Build json
+        working-directory: json
+        run: |
+          cmake -B build \
+            -DCMAKE_INSTALL_PREFIX=$HOME/json \
+            -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
+            -DJSON_BuildTests=OFF
+          cmake --build build --parallel 2
+          cmake --install build
+      - name: Checkout kokkos
+        uses: actions/checkout@v3
+        with:
+          repository: kokkos/kokkos
+          ref: ${{ matrix.kokkos_ver }}
+          path: kokkos
+      - name: Build kokkos
+        working-directory: kokkos
+        run: |
+          cmake -B build \
+            -DCMAKE_INSTALL_PREFIX=$HOME/kokkos \
+            -DKokkos_ENABLE_HIP=ON \
+            -DKokkos_ARCH_VEGA908=ON \
+            -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
+            -DCMAKE_CXX_COMPILER=${{ matrix.cxx }}
+          cmake --build build --parallel 2
+          cmake --install build
+      - name: Checkout Cabana
+        uses: actions/checkout@v3
+        with:
+          repository: ECP-CoPA/Cabana
+          # This version is post-release 0.5
+          ref: 31ba70d9e8f8e20f2e55e7e310905f950b21776f
+          path: cabana
+      - name: Build Cabana
+        working-directory: cabana
+        run: |
+          cmake -B build \
+            -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
+            -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \
+            -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \
+            -DCMAKE_PREFIX_PATH="$HOME/kokkos" \
+            -DCMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \
+            -DCMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
+            -DCMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
+            -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON \
+            -DCabana_REQUIRE_MPI=ON
+          cmake --build build --parallel 2
+          cmake --install build
+      - name: Checkout CabanaPD
+        uses: actions/checkout@v3
+      - name: Build CabanaPD
+        run: |
+          cmake -B build \
+            -D CMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
+            -D CMAKE_CXX_COMPILER=${{ matrix.cxx }} \
+            -D CMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \
+            -D CMAKE_INSTALL_PREFIX=$HOME/CabanaPD \
+            -D CMAKE_PREFIX_PATH="$HOME/Cabana;$HOME/json" \
+            -D CMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \
+            -D CMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
+            -D CMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
+            -D MPIEXEC_MAX_NUMPROCS=2 \
+            -D MPIEXEC_PREFLAGS="--oversubscribe" \
+            -D CabanaPD_ENABLE_TESTING=ON
+          cmake --build build --parallel 2
+          cmake --install build
+  CUDA:
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        cmake_build_type: ['Release']
+        # 3.7 needed for kokkos_compiler_launcher
+        kokkos_ver: ['3.7.02']
+    runs-on: ubuntu-20.04
+    container: ghcr.io/ecp-copa/ci-containers/cuda:12.2.0
+    steps:
+      - name: Checkout json
+        uses: actions/checkout@v3
+        with:
+          repository: nlohmann/json
+          ref: v3.11.2
+          path: json
+      - name: Build json
+        working-directory: json
+        run: |
+          cmake -B build \
+            -DCMAKE_INSTALL_PREFIX=$HOME/json \
+            -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
+            -DJSON_BuildTests=OFF
+          cmake --build build --parallel 2
+          cmake --install build
+      - name: Checkout kokkos
+        uses: actions/checkout@v3
+        with:
+          repository: kokkos/kokkos
+          ref: ${{ matrix.kokkos_ver }}
+          path: kokkos
+      - name: Build kokkos
+        working-directory: kokkos
+        run: |
+          cmake -B build \
+            -DCMAKE_INSTALL_PREFIX=$HOME/kokkos \
+            -DKokkos_ENABLE_CUDA=ON \
+            -DKokkos_ARCH_VOLTA72=ON \
+            -DKokkos_ENABLE_CUDA_LAMBDA=ON \
+            -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }}
+          cmake --build build --parallel 2
+          cmake --install build
+      - name: Checkout Cabana
+        uses: actions/checkout@v3
+        with:
+          repository: ECP-CoPA/Cabana
+          # This version is post-release 0.5
+          ref: 31ba70d9e8f8e20f2e55e7e310905f950b21776f
+          path: cabana
+      - name: Build Cabana
+        working-directory: cabana
+        run: |
+          cmake -B build \
+            -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
+            -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \
+            -DCMAKE_PREFIX_PATH="$HOME/kokkos" \
+            -DCMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \
+            -DCMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
+            -DCMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
+            -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON \
+            -DCabana_REQUIRE_MPI=ON
+          cmake --build build --parallel 2
+          cmake --install build
+      - name: Checkout CabanaPD
+        uses: actions/checkout@v3
+      - name: Build CabanaPD
+        run: |
+          cmake -B build \
+            -D CMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
+            -D CMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \
+            -D CMAKE_INSTALL_PREFIX=$HOME/CabanaPD \
+            -D CMAKE_PREFIX_PATH="$HOME/Cabana;$HOME/json" \
+            -D CMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \
+            -D CMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
+            -D CMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
+            -D MPIEXEC_MAX_NUMPROCS=2 \
+            -D MPIEXEC_PREFLAGS="--oversubscribe" \
+            -D CabanaPD_ENABLE_TESTING=ON
+          cmake --build build --parallel 2
+          cmake --install build
diff --git a/src/CabanaPD_Comm.hpp b/src/CabanaPD_Comm.hpp
@@ -80,9 +80,7 @@ struct HaloIds
         // balancing).
         neighborBounds( local_grid );
 
-        build(
-            positions,
-            KOKKOS_LAMBDA( const int, const double[3] ) { return true; } );
+        build( positions );
     }
 
     // Find the bounds of each neighbor rank and store for determining which
@@ -198,6 +196,16 @@ struct HaloIds
         Kokkos::fence();
     }
 
+    template <class PositionSliceType>
+    void build( const PositionSliceType& positions )
+    {
+        auto empty_functor = KOKKOS_LAMBDA( const int, const double[3] )
+        {
+            return true;
+        };
+        build( positions, empty_functor );
+    }
+
     template <class PositionSliceType>
     void rebuild( const PositionSliceType& positions )
     {
@@ -216,9 +224,7 @@ struct HaloIds
         if ( dest_count > dest_size )
         {
             Kokkos::deep_copy( _send_count, 0 );
-            build(
-                positions,
-                KOKKOS_LAMBDA( const int, const double[3] ) { return true; } );
+            build( positions );
         }
     }
 };