Grab updates from master

.jenkins

@@ -126,7 +126,7 @@ pipeline {
                             .. && \
                             make -j8 && \
                             # Exclude current failing tests (core MPI and partial Cajita)
-                            ctest --output-on-failure -E "Cabana.*MPI|Cabana_Performance_Comm|Cajita_IndexConversion|Cajita_Halo|Cajita_ParticleGridDistributor|Cajita_Interpolation|Cajita_BovWriter|valgrind"
+                            ctest --output-on-failure -E "Cabana.*MPI|Cabana_Performance_Comm|Grid_IndexConversion|Grid_Halo|Grid_ParticleGridDistributor|Grid_Interpolation|Grid_BovWriter|valgrind"
                         '''
                     }
                     post {

CHANGELOG.md

@@ -1,9 +1,38 @@
 # Change Log
 
-## (dev)
+## 0.6.0
 
-- Updated minimum Kokkos dependency to version 3.7
-- Updated heFFTe version from 2.1 to 2.3
+**New Features**
+ - C++17 required
+- `Gather`/`Scatter` for persistent particle communication buffers added
+- Grid-based particle generation, in addition to generalization of random particle generation
+- `ParticleList` with field type tags and thread-level particle interfaces added
+- Semi-structured hypre interface for multi-variate systems
+- New interface to modify an existing Verlet neighbor list
+- New benchmarks for grid halo and particle-grid interpolation
+
+**Bug Fixes and Improvements**
+- Breaking change: replace `Slice::{R->r}ank` and `Slice::{rank->viewRank}()`
+- Fixed boundary index space bug for `Node`, `Face`, and `Edge` entities causing out of bounds indexing
+- Added SYCL support for heFFTe
+- Removed use of Kokkos deprecated features: `View::Rank` and `parallel_reduce()` on raw pointers
+- Run benchmarks and examples with default or default host backends
+- Added Kokkos profiling regions
+- Added python utilities to plot benchmark results
+
+**Deprecated features**
+- Deprecated Cajita in favor of Cabana::Grid
+- Deprecated Kokkos `DeviceType` class templates in favor of Kokkos `MemorySpace`
+- Deprecated Grid `Halo` and `Partitioner` compatibility wrappers
+
+**Minimum dependency version updates**
+ - Kokkos minimum 3.7 (previously 3.2)
+ - Optional dependency heFFTe minimum 2.3 (previously 2.1)
+ - Optional dependency googletest minimum 1.10 (previous internal download removed)
+
+**Experimental Features (subject to change in future releases)**
+- Distributed particle output with HDF5 library interface
+- Expanded sparse grid capabilities: index spaces, local grid, array, and halo
 
 ## 0.5.0
 

CMakeLists.txt

@@ -44,21 +44,10 @@ set(CABANA_SUPPORTED_DEVICES SERIAL THREADS OPENMP CUDA HIP SYCL OPENMPTARGET)
 # check user required kokkos device types
 foreach(_device ${CABANA_SUPPORTED_DEVICES})
   option(Cabana_REQUIRE_${_device} "Build Cabana with required Kokkos ${device} support" OFF)
-  if( Cabana_REQUIRE_${_device} AND NOT _device STREQUAL THREADS)
+  if(Cabana_REQUIRE_${_device})
     kokkos_check( DEVICES ${_device} )
   endif()
 endforeach()
-# Threads workaround can go when we require a minimum Kokkos version of 3.6
-if(Cabana_REQUIRE_PTHREAD)
-  message(DEPRECATION "Cabana_REQUIRE_PTHREAD is deprecated, use Cabana_REQUIRE_THREADS instead!")
-endif()
-if(Cabana_REQUIRE_PTHREAD OR Cabana_REQUIRE_THREADS)
-  kokkos_check(DEVICES PTHREAD RETURN_VALUE KOKKOS_CHECK_PTHREAD_RETURN_VALUE)
-  kokkos_check(DEVICES THREADS RETURN_VALUE KOKKOS_CHECK_THREADS_RETURN_VALUE)
-  if(NOT KOKKOS_CHECK_PTHREAD_RETURN_VALUE AND NOT KOKKOS_CHECK_THREADS_RETURN_VALUE)
-    message(FATAL_ERROR "Kokkos does not provide the Threads backend requested")
-  endif()
-endif()
 
 # ensure that we can use lambdas
 if(Kokkos_ENABLE_CUDA)
@@ -89,7 +78,7 @@ Cabana_add_dependency( PACKAGE ArborX )
 Cabana_add_dependency( PACKAGE ALL )
 
 # find Clang Format
-find_package( CLANG_FORMAT 10 )
+find_package( CLANG_FORMAT 14 )
 
 # find hypre
 Cabana_add_dependency( PACKAGE HYPRE VERSION 2.22.1 )
@@ -230,7 +219,7 @@ endif()
 # enable doxygen
 find_package(Doxygen)
 if(Doxygen_FOUND)
-  doxygen_add_docs(doxygen core/src cajita/src)
+  doxygen_add_docs(doxygen core/src grid/src)
 endif()
 
 ##---------------------------------------------------------------------------##
@@ -239,12 +228,22 @@ endif()
 
 add_subdirectory(core)
 
-option(Cabana_ENABLE_CAJITA "Build Cajita (grid and particle-grid)" ${Cabana_ENABLE_MPI})
-if(Cabana_ENABLE_CAJITA)
+option(Cabana_ENABLE_GRID "Build grid and particle-grid capabilities" ${Cabana_ENABLE_MPI})
+### FIXME: remove in next release.
+option(Cabana_ENABLE_CAJITA "Build grid and particle-grid capabilities" OFF)
+option(Cabana_DISABLE_CAJITA_DEPRECATION_WARNINGS "Do not warn about Cajita deprecation." OFF)
+if (Cabana_ENABLE_CAJITA)
+  if (NOT Cabana_DISABLE_CAJITA_DEPRECATION_WARNINGS)
+    message(DEPRECATION "Option Cabana_ENABLE_CAJITA is deprecated, replaced by Cabana_ENABLE_GRID")
+  endif()
+  set(Cabana_ENABLE_GRID ON)
+endif()
+
+if(Cabana_ENABLE_GRID)
   if(MPI_FOUND)
-    add_subdirectory(cajita)
+    add_subdirectory(grid)
   else()
-    message(FATAL_ERROR "Cajita requires MPI")
+    message(FATAL_ERROR "Grid subpackage requires MPI")
   endif()
 endif()
 
@@ -278,7 +277,7 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/CabanaConfig.cmake" "${CMAKE_CURRENT_
 ## Clang Format
 ##---------------------------------------------------------------------------##
 if(CLANG_FORMAT_FOUND)
-  file(GLOB_RECURSE FORMAT_SOURCES core/*.cpp core/*.hpp cajita/*hpp cajita/*cpp example/*cpp example/*hpp cmake/*cpp cmake/*hpp benchmark/*cpp benchmark/*hpp)
+  file(GLOB_RECURSE FORMAT_SOURCES core/*.cpp core/*.hpp grid/*hpp grid/*cpp example/*cpp example/*hpp cmake/*cpp cmake/*hpp benchmark/*cpp benchmark/*hpp)
   add_custom_target(cabana-format
     COMMAND ${CLANG_FORMAT_EXECUTABLE} -i -style=file ${FORMAT_SOURCES}
     DEPENDS ${FORMAT_SOURCES})

README.md

@@ -1,13 +1,16 @@
-# CoPA Cabana - The Exascale Co-Design Center for Particle Applications Toolkit
+# Cabana
 
 Cabana is a performance portable library for particle-based simulations.
-Applications include, but are not limited to, Molecular Dynamics (MD) with
-short- and/or long-range interactions; various flavors of Particle-in-Cell
-(PIC) methods, including use within fluid and solid mechanics and plasma
-physics; and N-body cosmology simulations.
-Cabana provides particle data structures, algorithms, and utilities to enable
-simulations on a variety of platforms including many-core architectures and
-GPUs.
+Applications include, but are not limited to, molecular dynamics (MD) with
+short- and/or long-range atomic interactions; various flavors of particle-in-cell
+(PIC) methods, including use within fluid/solid mechanics and plasma
+physics; N-body cosmology simulations; and peridynamics for fracture mechanics.
+
+Cabana provides particle data structures, algorithms, and communication, as
+well as structured grids, grid algorithms, and particle-grid interpolation to
+enable simulations on a variety of platforms including many-core CPU and
+GPU architectures. Cabana is built on Kokkos, with many additional
+optional library dependencies, including MPI for multi-node simulation.
 
 Cabana is developed as part of the Co-Design Center for Particle Applications
 (CoPA) within the Exascale Computing Project (ECP) under the U.S. Department

benchmark/CMakeLists.txt

@@ -11,6 +11,6 @@
 
 add_subdirectory(core)
 
-if(Cabana_ENABLE_CAJITA)
-  add_subdirectory(cajita)
+if(Cabana_ENABLE_GRID)
+  add_subdirectory(grid)
 endif()

benchmark/core/CMakeLists.txt

@@ -10,22 +10,22 @@
 ############################################################################
 
 add_executable(BinSortPerformance Cabana_BinSortPerformance.cpp)
-target_link_libraries(BinSortPerformance cabanacore)
+target_link_libraries(BinSortPerformance Core)
 
 add_executable(NeighborVerletPerformance Cabana_NeighborVerletPerformance.cpp)
-target_link_libraries(NeighborVerletPerformance cabanacore)
+target_link_libraries(NeighborVerletPerformance Core)
 
 if(Cabana_ENABLE_ARBORX)
 add_executable(NeighborArborXPerformance Cabana_NeighborArborXPerformance.cpp)
-target_link_libraries(NeighborArborXPerformance cabanacore)
+target_link_libraries(NeighborArborXPerformance Core)
 endif()
 
 add_executable(LinkedCellPerformance Cabana_LinkedCellPerformance.cpp)
-target_link_libraries(LinkedCellPerformance cabanacore)
+target_link_libraries(LinkedCellPerformance Core)
 
 if(Cabana_ENABLE_MPI)
 add_executable(CommPerformance Cabana_CommPerformance.cpp)
-target_link_libraries(CommPerformance cabanacore)
+target_link_libraries(CommPerformance Core)
 endif()
 
 if(Cabana_ENABLE_TESTING)

benchmark/core/Cabana_BinSortPerformance.cpp

@@ -31,11 +31,13 @@ void performanceTest( std::ostream& stream, const std::string& test_prefix,
                       std::vector<int> problem_sizes,
                       std::vector<int> num_bins )
 {
+    using memory_space = typename Device::memory_space;
+
     // Declare problem sizes.
     int num_problem_size = problem_sizes.size();
 
     // Generate a random set of keys
-    Kokkos::View<unsigned long*, Device> keys(
+    Kokkos::View<unsigned long*, memory_space> keys(
         Kokkos::ViewAllocateWithoutInitializing( "keys" ),
         problem_sizes.back() );
     Kokkos::View<unsigned long*, Kokkos::HostSpace> host_keys(
@@ -54,7 +56,7 @@ void performanceTest( std::ostream& stream, const std::string& test_prefix,
 
     // Define the aosoa.
     using member_types = Cabana::MemberTypes<double[3], double[3], double, int>;
-    using aosoa_type = Cabana::AoSoA<member_types, Device>;
+    using aosoa_type = Cabana::AoSoA<member_types, memory_space>;
 
     // Create aosoas.
     std::vector<aosoa_type> aosoas( num_problem_size );

benchmark/core/Cabana_LinkedCellPerformance.cpp

@@ -30,6 +30,8 @@ void performanceTest( std::ostream& stream, const std::string& test_prefix,
                       std::vector<int> problem_sizes,
                       std::vector<double> cutoff_ratios )
 {
+    using memory_space = typename Device::memory_space;
+
     // Declare problem sizes.
     int num_problem_size = problem_sizes.size();
     std::vector<double> x_min( num_problem_size );
@@ -44,7 +46,7 @@ void performanceTest( std::ostream& stream, const std::string& test_prefix,
 
     // Define the aosoa.
     using member_types = Cabana::MemberTypes<double[3]>;
-    using aosoa_type = Cabana::AoSoA<member_types, Device>;
+    using aosoa_type = Cabana::AoSoA<member_types, memory_space>;
     std::vector<aosoa_type> aosoas( num_problem_size );
 
     // Create aosoas.
@@ -55,9 +57,12 @@ void performanceTest( std::ostream& stream, const std::string& test_prefix,
         // Define problem grid.
         x_min[p] = 0.0;
         x_max[p] = 1.3 * std::pow( num_p, 1.0 / 3.0 );
+        double grid_min[3] = { x_min[p], x_min[p], x_min[p] };
+        double grid_max[3] = { x_max[p], x_max[p], x_max[p] };
         aosoas[p].resize( num_p );
         auto x = Cabana::slice<0>( aosoas[p], "position" );
-        Cabana::createRandomParticles( x, x.size(), x_min[p], x_max[p] );
+        Cabana::createParticles( Cabana::InitRandom(), x, x.size(), grid_min,
+                                 grid_max );
     }
 
     // Loop over number of ratios (neighbors per particle).
@@ -93,7 +98,7 @@ void performanceTest( std::ostream& stream, const std::string& test_prefix,
             double sort_delta[3] = { cutoff, cutoff, cutoff };
             double grid_min[3] = { x_min[p], x_min[p], x_min[p] };
             double grid_max[3] = { x_max[p], x_max[p], x_max[p] };
-            Cabana::LinkedCellList<Device> linked_cell_list(
+            Cabana::LinkedCellList<memory_space> linked_cell_list(
                 x, sort_delta, grid_min, grid_max );
 
             // Run tests and time the ensemble

benchmark/core/Cabana_NeighborArborXPerformance.cpp

@@ -51,7 +51,7 @@ void performanceTest( std::ostream& stream, const std::string& test_prefix,
 
     // Define the aosoa.
     using member_types = Cabana::MemberTypes<double[3]>;
-    using aosoa_type = Cabana::AoSoA<member_types, Device>;
+    using aosoa_type = Cabana::AoSoA<member_types, memory_space>;
     std::vector<aosoa_type> aosoas( num_problem_size );
 
     // Create aosoas.
@@ -62,9 +62,12 @@ void performanceTest( std::ostream& stream, const std::string& test_prefix,
         // Define problem grid.
         x_min[p] = 0.0;
         x_max[p] = 1.3 * std::pow( num_p, 1.0 / 3.0 );
+        double grid_min[3] = { x_min[p], x_min[p], x_min[p] };
+        double grid_max[3] = { x_max[p], x_max[p], x_max[p] };
         aosoas[p].resize( num_p );
         auto x = Cabana::slice<0>( aosoas[p], "position" );
-        Cabana::createRandomParticles( x, x.size(), x_min[p], x_max[p] );
+        Cabana::createParticles( Cabana::InitRandom(), x, x.size(), grid_min,
+                                 grid_max );
 
         if ( sort )
         {
@@ -74,10 +77,8 @@ void performanceTest( std::ostream& stream, const std::string& test_prefix,
             // in cells the size of the smallest cutoff distance.
             double cutoff = cutoff_ratios.front();
             double sort_delta[3] = { cutoff, cutoff, cutoff };
-            double grid_min[3] = { x_min[p], x_min[p], x_min[p] };
-            double grid_max[3] = { x_max[p], x_max[p], x_max[p] };
             auto x = Cabana::slice<0>( aosoas[p], "position" );
-            Cabana::LinkedCellList<Device> linked_cell_list(
+            Cabana::LinkedCellList<memory_space> linked_cell_list(
                 x, sort_delta, grid_min, grid_max );
             Cabana::permute( linked_cell_list, aosoas[p] );
         }
@@ -126,10 +127,9 @@ void performanceTest( std::ostream& stream, const std::string& test_prefix,
                 // Create the neighbor list.
                 double cutoff = cutoff_ratios[c];
                 create_timer.start( pid );
-                auto const nlist =
-                    Cabana::Experimental::make2DNeighborList<Device>(
-                        ListTag{}, Cabana::slice<0>( aosoas[p], "position" ), 0,
-                        num_p, cutoff );
+                auto const nlist = Cabana::Experimental::make2DNeighborList(
+                    ListTag{}, Cabana::slice<0>( aosoas[p], "position" ), 0,
+                    num_p, cutoff );
                 create_timer.stop( pid );
 
                 // Iterate through the neighbor list.