Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cuda build CI #61

Merged
merged 4 commits into from
Oct 16, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 0 additions & 99 deletions .github/workflows/CI-e4s-build.yml

This file was deleted.

175 changes: 172 additions & 3 deletions .github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
container: ghcr.io/ecp-copa/ci-containers/${{ matrix.distro }}
steps:
- name: Checkout kokkos
uses: actions/checkout@v2.2.0
uses: actions/checkout@v3
with:
repository: kokkos/kokkos
ref: ${{ matrix.kokkos_ver }}
Expand All @@ -62,7 +62,7 @@ jobs:
cmake --build build --parallel 2
cmake --install build
- name: Checkout Cabana
uses: actions/checkout@v2.2.0
uses: actions/checkout@v3
with:
repository: ECP-CoPA/Cabana
# This version is post-release 0.5
Expand All @@ -89,7 +89,7 @@ jobs:
cmake --build build --parallel 2
cmake --install build
- name: Checkout CabanaPD
uses: actions/checkout@v2.2.0
uses: actions/checkout@v3
- name: Build CabanaPD
run: |
cmake -B build \
Expand All @@ -106,3 +106,172 @@ jobs:
- name: Test CabanaPD
run: |
CTEST_OUTPUT_ON_FAILURE=1 cmake --build build --target test
HIP:
defaults:
run:
shell: bash
env:
# Needed because FindMPI relies heavily on the compiler wrappers, which aren't used with hipcc
MPI_LOCATION: /usr/lib/x86_64-linux-gnu/openmpi/
strategy:
matrix:
cxx: ['hipcc']
cmake_build_type: ['Release']
kokkos_ver: ['3.6.01']
runs-on: ubuntu-20.04
container: ghcr.io/ecp-copa/ci-containers/rocm:latest
steps:
- name: Checkout json
uses: actions/checkout@v3
with:
repository: nlohmann/json
ref: v3.11.2
path: json
- name: Build json
working-directory: json
run: |
cmake -B build \
-DCMAKE_INSTALL_PREFIX=$HOME/json \
-DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
-DJSON_BuildTests=OFF
cmake --build build --parallel 2
cmake --install build
- name: Checkout kokkos
uses: actions/checkout@v3
with:
repository: kokkos/kokkos
ref: ${{ matrix.kokkos_ver }}
path: kokkos
- name: Build kokkos
working-directory: kokkos
run: |
cmake -B build \
-DCMAKE_INSTALL_PREFIX=$HOME/kokkos \
-DKokkos_ENABLE_HIP=ON \
-DKokkos_ARCH_VEGA908=ON \
-DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
-DCMAKE_CXX_COMPILER=${{ matrix.cxx }}
cmake --build build --parallel 2
cmake --install build
- name: Checkout Cabana
uses: actions/checkout@v3
with:
repository: ECP-CoPA/Cabana
# This version is post-release 0.5
ref: 31ba70d9e8f8e20f2e55e7e310905f950b21776f
path: cabana
- name: Build Cabana
working-directory: cabana
run: |
cmake -B build \
-DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
-DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \
-DCMAKE_INSTALL_PREFIX=$HOME/Cabana \
-DCMAKE_PREFIX_PATH="$HOME/kokkos" \
-DCMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \
-DCMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
-DCMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
-DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON \
-DCabana_REQUIRE_MPI=ON
cmake --build build --parallel 2
cmake --install build
- name: Checkout CabanaPD
uses: actions/checkout@v3
- name: Build CabanaPD
run: |
cmake -B build \
-D CMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
-D CMAKE_CXX_COMPILER=${{ matrix.cxx }} \
-D CMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \
-D CMAKE_INSTALL_PREFIX=$HOME/CabanaPD \
-D CMAKE_PREFIX_PATH="$HOME/Cabana;$HOME/json" \
-D CMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \
-D CMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
-D CMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
-D MPIEXEC_MAX_NUMPROCS=2 \
-D MPIEXEC_PREFLAGS="--oversubscribe" \
-D CabanaPD_ENABLE_TESTING=ON
cmake --build build --parallel 2
cmake --install build
CUDA:
defaults:
run:
shell: bash
strategy:
matrix:
cmake_build_type: ['Release']
# 3.7 needed for kokkos_compiler_launcher
kokkos_ver: ['3.7.02']
runs-on: ubuntu-20.04
container: ghcr.io/ecp-copa/ci-containers/cuda:12.2.0
steps:
- name: Checkout json
uses: actions/checkout@v3
with:
repository: nlohmann/json
ref: v3.11.2
path: json
- name: Build json
working-directory: json
run: |
cmake -B build \
-DCMAKE_INSTALL_PREFIX=$HOME/json \
-DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
-DJSON_BuildTests=OFF
cmake --build build --parallel 2
cmake --install build
- name: Checkout kokkos
uses: actions/checkout@v3
with:
repository: kokkos/kokkos
ref: ${{ matrix.kokkos_ver }}
path: kokkos
- name: Build kokkos
working-directory: kokkos
run: |
cmake -B build \
-DCMAKE_INSTALL_PREFIX=$HOME/kokkos \
-DKokkos_ENABLE_CUDA=ON \
-DKokkos_ARCH_VOLTA72=ON \
-DKokkos_ENABLE_CUDA_LAMBDA=ON \
-DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }}
cmake --build build --parallel 2
cmake --install build
- name: Checkout Cabana
uses: actions/checkout@v3
with:
repository: ECP-CoPA/Cabana
# This version is post-release 0.5
ref: 31ba70d9e8f8e20f2e55e7e310905f950b21776f
path: cabana
- name: Build Cabana
working-directory: cabana
run: |
cmake -B build \
-DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
-DCMAKE_INSTALL_PREFIX=$HOME/Cabana \
-DCMAKE_PREFIX_PATH="$HOME/kokkos" \
-DCMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \
-DCMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
-DCMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
-DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON \
-DCabana_REQUIRE_MPI=ON
cmake --build build --parallel 2
cmake --install build
- name: Checkout CabanaPD
uses: actions/checkout@v3
- name: Build CabanaPD
run: |
cmake -B build \
-D CMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
-D CMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \
-D CMAKE_INSTALL_PREFIX=$HOME/CabanaPD \
-D CMAKE_PREFIX_PATH="$HOME/Cabana;$HOME/json" \
-D CMAKE_CXX_FLAGS="-I${MPI_LOCATION}/include" \
-D CMAKE_EXE_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
-D CMAKE_SHARED_LINKER_FLAGS="-L${MPI_LOCATION}/lib -lmpi" \
-D MPIEXEC_MAX_NUMPROCS=2 \
-D MPIEXEC_PREFLAGS="--oversubscribe" \
-D CabanaPD_ENABLE_TESTING=ON
cmake --build build --parallel 2
cmake --install build
18 changes: 12 additions & 6 deletions src/CabanaPD_Comm.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,7 @@ struct HaloIds
// balancing).
neighborBounds( local_grid );

build(
positions,
KOKKOS_LAMBDA( const int, const double[3] ) { return true; } );
build( positions );
}

// Find the bounds of each neighbor rank and store for determining which
Expand Down Expand Up @@ -198,6 +196,16 @@ struct HaloIds
Kokkos::fence();
}

template <class PositionSliceType>
void build( const PositionSliceType& positions )
{
auto empty_functor = KOKKOS_LAMBDA( const int, const double[3] )
{
return true;
};
build( positions, empty_functor );
}

template <class PositionSliceType>
void rebuild( const PositionSliceType& positions )
{
Expand All @@ -216,9 +224,7 @@ struct HaloIds
if ( dest_count > dest_size )
{
Kokkos::deep_copy( _send_count, 0 );
build(
positions,
KOKKOS_LAMBDA( const int, const double[3] ) { return true; } );
build( positions );
}
}
};
Expand Down
Loading