From 0a99fe8604db22b1d6cd04a18c0b8277e5f9fd06 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Fri, 6 Oct 2023 11:12:39 -0400 Subject: [PATCH 1/4] Move all CI to one file --- .github/workflows/CI-e4s-build.yml | 99 ------------------------------ .github/workflows/CI.yml | 87 ++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 99 deletions(-) delete mode 100644 .github/workflows/CI-e4s-build.yml diff --git a/.github/workflows/CI-e4s-build.yml b/.github/workflows/CI-e4s-build.yml deleted file mode 100644 index bab3ffda..00000000 --- a/.github/workflows/CI-e4s-build.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: e4s-build -on: - push: - branches: - - main - pull_request: - branches: - - main -concurrency: - group: ${ {github.event_name }}-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{github.event_name == 'pull_request'}} -jobs: - HIP: - defaults: - run: - shell: bash - env: - # Needed because FindMPI relies heavily on the compiler wrappers, which aren't used with hipcc - MPI_LOCATION: /usr/lib/x86_64-linux-gnu/openmpi/ - strategy: - matrix: - cxx: ['hipcc'] - cmake_build_type: ['Release'] - kokkos_ver: ['3.6.01'] - runs-on: ubuntu-20.04 - container: ghcr.io/ecp-copa/ci-containers/rocm:latest - steps: - - name: Checkout json - uses: actions/checkout@v3 - with: - repository: nlohmann/json - ref: v3.11.2 - path: json - - name: Build json - working-directory: json - run: | - cmake -B build \ - -DCMAKE_INSTALL_PREFIX=$HOME/json \ - -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ - -DJSON_BuildTests=OFF - cmake --build build --parallel 2 - cmake --install build - - name: Checkout kokkos - uses: actions/checkout@v3 - with: - repository: kokkos/kokkos - ref: ${{ matrix.kokkos_ver }} - path: kokkos - - name: Build kokkos - working-directory: kokkos - run: | - cmake -B build \ - -DCMAKE_INSTALL_PREFIX=$HOME/kokkos \ - -DKokkos_ENABLE_HIP=ON \ - -DKokkos_ARCH_VEGA908=ON \ - -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ - -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} - cmake --build build --parallel 2 - cmake --install build - - name: Checkout Cabana - uses: actions/checkout@v2.2.0 - with: - repository: ECP-CoPA/Cabana - # This version is post-release 0.5 - ref: 31ba70d9e8f8e20f2e55e7e310905f950b21776f - path: cabana - - name: Build Cabana - working-directory: cabana - run: | - cmake -B build \ - -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ - -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ - -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \ - -DCMAKE_PREFIX_PATH="$HOME/kokkos" \ - -DCMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \ - -DCMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ - -DCMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ - -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON \ - -DCabana_REQUIRE_MPI=ON - cmake --build build --parallel 2 - cmake --install build - - name: Checkout CabanaPD - uses: actions/checkout@v3 - - name: Build CabanaPD - run: | - cmake -B build \ - -D CMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ - -D CMAKE_CXX_COMPILER=${{ matrix.cxx }} \ - -D CMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \ - -D CMAKE_INSTALL_PREFIX=$HOME/CabanaPD \ - -D CMAKE_PREFIX_PATH="$HOME/Cabana;$HOME/json" \ - -D CMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \ - -D CMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ - -D CMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ - -D MPIEXEC_MAX_NUMPROCS=2 \ - -D MPIEXEC_PREFLAGS="--oversubscribe" \ - -D CabanaPD_ENABLE_TESTING=ON - cmake --build build --parallel 2 - cmake --install build diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ba5ec39a..e8978380 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -106,3 +106,90 @@ jobs: - name: Test CabanaPD run: | CTEST_OUTPUT_ON_FAILURE=1 cmake --build build --target test + HIP: + defaults: + run: + shell: bash + env: + # Needed because FindMPI relies heavily on the compiler wrappers, which aren't used with hipcc + MPI_LOCATION: /usr/lib/x86_64-linux-gnu/openmpi/ + strategy: + matrix: + cxx: ['hipcc'] + cmake_build_type: ['Release'] + kokkos_ver: ['3.6.01'] + runs-on: ubuntu-20.04 + container: ghcr.io/ecp-copa/ci-containers/rocm:latest + steps: + - name: Checkout json + uses: actions/checkout@v3 + with: + repository: nlohmann/json + ref: v3.11.2 + path: json + - name: Build json + working-directory: json + run: | + cmake -B build \ + -DCMAKE_INSTALL_PREFIX=$HOME/json \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ + -DJSON_BuildTests=OFF + cmake --build build --parallel 2 + cmake --install build + - name: Checkout kokkos + uses: actions/checkout@v3 + with: + repository: kokkos/kokkos + ref: ${{ matrix.kokkos_ver }} + path: kokkos + - name: Build kokkos + working-directory: kokkos + run: | + cmake -B build \ + -DCMAKE_INSTALL_PREFIX=$HOME/kokkos \ + -DKokkos_ENABLE_HIP=ON \ + -DKokkos_ARCH_VEGA908=ON \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} + cmake --build build --parallel 2 + cmake --install build + - name: Checkout Cabana + uses: actions/checkout@v3 + with: + repository: ECP-CoPA/Cabana + # This version is post-release 0.5 + ref: 31ba70d9e8f8e20f2e55e7e310905f950b21776f + path: cabana + - name: Build Cabana + working-directory: cabana + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ + -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \ + -DCMAKE_PREFIX_PATH="$HOME/kokkos" \ + -DCMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \ + -DCMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ + -DCMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ + -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON \ + -DCabana_REQUIRE_MPI=ON + cmake --build build --parallel 2 + cmake --install build + - name: Checkout CabanaPD + uses: actions/checkout@v3 + - name: Build CabanaPD + run: | + cmake -B build \ + -D CMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ + -D CMAKE_CXX_COMPILER=${{ matrix.cxx }} \ + -D CMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \ + -D CMAKE_INSTALL_PREFIX=$HOME/CabanaPD \ + -D CMAKE_PREFIX_PATH="$HOME/Cabana;$HOME/json" \ + -D CMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \ + -D CMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ + -D CMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ + -D MPIEXEC_MAX_NUMPROCS=2 \ + -D MPIEXEC_PREFLAGS="--oversubscribe" \ + -D CabanaPD_ENABLE_TESTING=ON + cmake --build build --parallel 2 + cmake --install build From 26fb14f0aab06f34f4d87724860379a39681435c Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Fri, 6 Oct 2023 11:12:57 -0400 Subject: [PATCH 2/4] fixup: use actions v3 --- .github/workflows/CI.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e8978380..3f8f438b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -45,7 +45,7 @@ jobs: container: ghcr.io/ecp-copa/ci-containers/${{ matrix.distro }} steps: - name: Checkout kokkos - uses: actions/checkout@v2.2.0 + uses: actions/checkout@v3 with: repository: kokkos/kokkos ref: ${{ matrix.kokkos_ver }} @@ -62,7 +62,7 @@ jobs: cmake --build build --parallel 2 cmake --install build - name: Checkout Cabana - uses: actions/checkout@v2.2.0 + uses: actions/checkout@v3 with: repository: ECP-CoPA/Cabana # This version is post-release 0.5 @@ -89,7 +89,7 @@ jobs: cmake --build build --parallel 2 cmake --install build - name: Checkout CabanaPD - uses: actions/checkout@v2.2.0 + uses: actions/checkout@v3 - name: Build CabanaPD run: | cmake -B build \ From e68f0d69f70aeb2abd5f6c2cd7ba3b2bfcbbe038 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:29:47 -0400 Subject: [PATCH 3/4] Add CUDA build CI --- .github/workflows/CI.yml | 82 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 3f8f438b..be203ca9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -193,3 +193,85 @@ jobs: -D CabanaPD_ENABLE_TESTING=ON cmake --build build --parallel 2 cmake --install build + CUDA: + defaults: + run: + shell: bash + strategy: + matrix: + cmake_build_type: ['Release'] + # 3.7 needed for kokkos_compiler_launcher + kokkos_ver: ['3.7.02'] + runs-on: ubuntu-20.04 + container: ghcr.io/ecp-copa/ci-containers/cuda:12.2.0 + steps: + - name: Checkout json + uses: actions/checkout@v3 + with: + repository: nlohmann/json + ref: v3.11.2 + path: json + - name: Build json + working-directory: json + run: | + cmake -B build \ + -DCMAKE_INSTALL_PREFIX=$HOME/json \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ + -DJSON_BuildTests=OFF + cmake --build build --parallel 2 + cmake --install build + - name: Checkout kokkos + uses: actions/checkout@v3 + with: + repository: kokkos/kokkos + ref: ${{ matrix.kokkos_ver }} + path: kokkos + - name: Build kokkos + working-directory: kokkos + run: | + cmake -B build \ + -DCMAKE_INSTALL_PREFIX=$HOME/kokkos \ + -DKokkos_ENABLE_CUDA=ON \ + -DKokkos_ARCH_VOLTA72=ON \ + -DKokkos_ENABLE_CUDA_LAMBDA=ON \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} + cmake --build build --parallel 2 + cmake --install build + - name: Checkout Cabana + uses: actions/checkout@v3 + with: + repository: ECP-CoPA/Cabana + # This version is post-release 0.5 + ref: 31ba70d9e8f8e20f2e55e7e310905f950b21776f + path: cabana + - name: Build Cabana + working-directory: cabana + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ + -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \ + -DCMAKE_PREFIX_PATH="$HOME/kokkos" \ + -DCMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \ + -DCMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ + -DCMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ + -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON \ + -DCabana_REQUIRE_MPI=ON + cmake --build build --parallel 2 + cmake --install build + - name: Checkout CabanaPD + uses: actions/checkout@v3 + - name: Build CabanaPD + run: | + cmake -B build \ + -D CMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \ + -D CMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \ + -D CMAKE_INSTALL_PREFIX=$HOME/CabanaPD \ + -D CMAKE_PREFIX_PATH="$HOME/Cabana;$HOME/json" \ + -D CMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \ + -D CMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ + -D CMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \ + -D MPIEXEC_MAX_NUMPROCS=2 \ + -D MPIEXEC_PREFLAGS="--oversubscribe" \ + -D CabanaPD_ENABLE_TESTING=ON + cmake --build build --parallel 2 + cmake --install build From 14005e1e1ae15a451003ac6183842f3133648377 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:55:20 -0400 Subject: [PATCH 4/4] fixup: halo capture for cuda --- src/CabanaPD_Comm.hpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/CabanaPD_Comm.hpp b/src/CabanaPD_Comm.hpp index fa6f443b..ea91a415 100644 --- a/src/CabanaPD_Comm.hpp +++ b/src/CabanaPD_Comm.hpp @@ -80,9 +80,7 @@ struct HaloIds // balancing). neighborBounds( local_grid ); - build( - positions, - KOKKOS_LAMBDA( const int, const double[3] ) { return true; } ); + build( positions ); } // Find the bounds of each neighbor rank and store for determining which @@ -198,6 +196,16 @@ struct HaloIds Kokkos::fence(); } + template + void build( const PositionSliceType& positions ) + { + auto empty_functor = KOKKOS_LAMBDA( const int, const double[3] ) + { + return true; + }; + build( positions, empty_functor ); + } + template void rebuild( const PositionSliceType& positions ) { @@ -216,9 +224,7 @@ struct HaloIds if ( dest_count > dest_size ) { Kokkos::deep_copy( _send_count, 0 ); - build( - positions, - KOKKOS_LAMBDA( const int, const double[3] ) { return true; } ); + build( positions ); } } };