Skip to content

Commit

Permalink
Clarify offset vs number of each type of particles
Browse files Browse the repository at this point in the history
  • Loading branch information
@streeve
streeve committed Dec 17, 2024
1 parent b263b6b commit 391444e
Show file tree
Hide file tree
Showing 15 changed files with 126 additions and 123 deletions.
2 changes: 1 addition & 1 deletion examples/mechanics/fragmenting_cylinder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ void fragmentingCylinderExample( const std::string filename )
using random_type = Kokkos::Random_XorShift64<exec_space>;
pool_type pool;
int seed = 456854;
pool.init( seed, particles->n_local );
pool.init( seed, particles->localOffset() );
auto init_functor = KOKKOS_LAMBDA( const int pid )
{
// Density
Expand Down
4 changes: 2 additions & 2 deletions src/CabanaPD_BodyTerm.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ struct BodyTerm
void apply( ExecSpace, ParticleType& particles, const double time )
{
_timer.start();
Kokkos::RangePolicy<ExecSpace> policy( particles.numFrozen(),
particles.numLocal() );
Kokkos::RangePolicy<ExecSpace> policy( particles.frozenOffset(),
particles.localOffset() );
auto user = _user_functor;
Kokkos::parallel_for(
"CabanaPD::BodyTerm::apply", policy,
Expand Down
4 changes: 2 additions & 2 deletions src/CabanaPD_Boundary.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ struct BoundaryIndexSpace<MemorySpace, RegionBoundary<GeometryType>>
_timer.start();

_view = index_view_type( "boundary_indices",
particles.numLocal() * initial_guess );
particles.localOffset() * initial_guess );
_count = index_view_type( "count", 1 );

for ( RegionBoundary plane : planes )
Expand Down Expand Up @@ -169,7 +169,7 @@ struct BoundaryIndexSpace<MemorySpace, RegionBoundary<GeometryType>>
auto index_space = _view;
auto count = _count;
auto x = particles.sliceReferencePosition();
Kokkos::RangePolicy<ExecSpace> policy( 0, particles.numLocal() );
Kokkos::RangePolicy<ExecSpace> policy( 0, particles.localOffset() );
auto index_functor = KOKKOS_LAMBDA( const std::size_t pid )
{
if ( region.inside( x, pid ) )
Expand Down
6 changes: 3 additions & 3 deletions src/CabanaPD_Comm.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,9 +270,9 @@ class Comm<ParticleType, PMB, TemperatureIndependent>
halo_ids.rebuild( positions );

// Create the Cabana Halo.
halo = std::make_shared<halo_type>( local_grid->globalGrid().comm(),
particles.numLocal(), halo_ids._ids,
halo_ids._destinations, topology );
halo = std::make_shared<halo_type>(
local_grid->globalGrid().comm(), particles.localOffset(),
halo_ids._ids, halo_ids._destinations, topology );

particles.resize( halo->numLocal(), halo->numGhost() );

Expand Down
3 changes: 2 additions & 1 deletion src/CabanaPD_DisplacementProfile.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,9 @@ void createOutputProfile( MPI_Comm comm, const int num_cell,
profile( c, 1 ) = user( pid );
}
};
// TODO: enable ignoring frozen particles.
Kokkos::RangePolicy<typename memory_space::execution_space> policy(
0, particles.numLocal() );
0, particles.localOffset() );
Kokkos::parallel_for( "displacement_profile", policy, measure_profile );
auto count_host =
Kokkos::create_mirror_view_and_copy( Kokkos::HostSpace{}, count );
Expand Down
4 changes: 4 additions & 0 deletions src/CabanaPD_Force.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,8 @@ class Force<MemorySpace, BaseForceModel>
Force( const bool half_neigh, const double delta,
const ParticleType& particles, const double tol = 1e-14 )
: _half_neigh( half_neigh )
// Note: this construction must always include frozen particles as
// potential neighbors, so we start at zero.
, _neigh_list( neighbor_list_type( particles.sliceReferencePosition(),
0, particles.numLocal(), delta + tol,
1.0, particles.ghost_mesh_lo,
Expand All @@ -164,6 +166,8 @@ class Force<MemorySpace, BaseForceModel>
const double mesh_min[3], const double mesh_max[3],
const double tol = 1e-14 )
: _half_neigh( half_neigh )
// Note: this construction must always include frozen particles as
// potential neighbors, so we start at zero.
, _neigh_list( neighbor_list_type( positions, 0, num_local, delta + tol,
1.0, mesh_min, mesh_max ) )
{
Expand Down
8 changes: 4 additions & 4 deletions src/CabanaPD_HeatTransfer.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ class HeatTransfer : public Force<MemorySpace, BaseForceModel>
coeff * ( temp( j ) - temp( i ) ) / xi / xi * vol( j );
};

Kokkos::RangePolicy<exec_space> policy( particles.numFrozen(),
particles.numLocal() );
Kokkos::RangePolicy<exec_space> policy( particles.frozenOffset(),
particles.localOffset() );
Cabana::neighbor_parallel_for(
policy, temp_func, _neigh_list, Cabana::FirstNeighborsTag(),
neigh_op_tag, "CabanaPD::HeatTransfer::computeFull" );
Expand All @@ -91,8 +91,8 @@ class HeatTransfer : public Force<MemorySpace, BaseForceModel>
{
temp( i ) += dt / rho( i ) / model.cp * conduction( i );
};
Kokkos::RangePolicy<exec_space> policy( particles.numFrozen(),
particles.numLocal() );
Kokkos::RangePolicy<exec_space> policy( particles.frozenOffset(),
particles.localOffset() );
Kokkos::parallel_for( "CabanaPD::HeatTransfer::forwardEuler", policy,
euler_func );
_euler_timer.stop();
Expand Down
6 changes: 4 additions & 2 deletions src/CabanaPD_Integrate.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ class Integrator
u( i, 1 ) += dt * v( i, 1 );
u( i, 2 ) += dt * v( i, 2 );
};
Kokkos::RangePolicy<exec_space> policy( p.numFrozen(), p.numLocal() );
Kokkos::RangePolicy<exec_space> policy( p.frozenOffset(),
p.localOffset() );
Kokkos::parallel_for( "CabanaPD::Integrator::Initial", policy,
init_func );

Expand All @@ -130,7 +131,8 @@ class Integrator
v( i, 1 ) += half_dt_m * f( i, 1 );
v( i, 2 ) += half_dt_m * f( i, 2 );
};
Kokkos::RangePolicy<exec_space> policy( p.numFrozen(), p.numLocal() );
Kokkos::RangePolicy<exec_space> policy( p.frozenOffset(),
p.localOffset() );
Kokkos::parallel_for( "CabanaPD::Integrator::Final", policy,
final_func );

Expand Down
118 changes: 56 additions & 62 deletions src/CabanaPD_Particles.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,10 @@ class Particles<MemorySpace, PMB, TemperatureIndependent, BaseOutput, Dimension>
static constexpr int dim = Dimension;

// Per particle.
unsigned long long int n_global = 0;
std::size_t n_local = 0;
std::size_t n_frozen = 0;
std::size_t n_ghost = 0;
unsigned long long int num_global = 0;
std::size_t frozen_offset = 0;
std::size_t local_offset = 0;
std::size_t num_ghost = 0;
std::size_t size = 0;

// x, u, f (vector matching system dimension).
Expand Down Expand Up @@ -239,7 +239,7 @@ class Particles<MemorySpace, PMB, TemperatureIndependent, BaseOutput, Dimension>
{
return true;
};
createParticles( exec_space, empty );
createParticles( exec_space, empty, create_frozen );
}

template <class ExecSpace, class UserFunctor>
Expand Down Expand Up @@ -297,39 +297,44 @@ class Particles<MemorySpace, PMB, TemperatureIndependent, BaseOutput, Dimension>

return create;
};
n_local = Cabana::Grid::createParticles( Cabana::InitUniform{},
exec_space, create_functor,
_plist_x, 1, *local_grid );
resize( n_local, 0 );
local_offset = Cabana::Grid::createParticles(
Cabana::InitUniform{}, exec_space, create_functor, _plist_x, 1,
*local_grid );
resize( local_offset, 0 );
size = _plist_x.size();

// Only set this value if this generation of particles should be frozen.
if ( create_frozen )
n_frozen = size;
frozen_offset = size;

// Not using Allreduce because global count is only used for printing.
auto n_local_mpi = static_cast<unsigned long long int>( n_local );
MPI_Reduce( &n_local_mpi, &n_global, 1, MPI_UNSIGNED_LONG_LONG, MPI_SUM,
0, MPI_COMM_WORLD );
auto local_offset_mpi =
static_cast<unsigned long long int>( local_offset );
MPI_Reduce( &local_offset_mpi, &num_global, 1, MPI_UNSIGNED_LONG_LONG,
MPI_SUM, 0, MPI_COMM_WORLD );
_init_timer.stop();
}

template <class ExecSpace, class FunctorType>
void updateParticles( const ExecSpace, const FunctorType init_functor )
{
_timer.start();
Kokkos::RangePolicy<ExecSpace> policy( 0, n_local );
Kokkos::RangePolicy<ExecSpace> policy( 0, local_offset );
Kokkos::parallel_for(
"CabanaPD::Particles::update_particles", policy,
KOKKOS_LAMBDA( const int pid ) { init_functor( pid ); } );
_timer.stop();
}

auto numFrozen() const { return n_frozen; }
auto numLocal() const { return n_local; }
auto numGhost() const { return n_ghost; }
auto numReference() const { return size; }
auto numGlobal() const { return n_global; }
// Particles are always in order frozen, local, ghost.
// Values for offsets are distinguished from separate (num) values.
auto numFrozen() const { return frozen_offset; }
auto frozenOffset() const { return frozen_offset; }
auto numLocal() const { return local_offset - frozen_offset; }
auto localOffset() const { return local_offset; }
auto numGhost() const { return num_ghost; }
auto referenceOffset() const { return size; }
auto numGlobal() const { return num_global; }

auto sliceReferencePosition()
{
Expand Down Expand Up @@ -408,7 +413,8 @@ class Particles<MemorySpace, PMB, TemperatureIndependent, BaseOutput, Dimension>
auto y = Cabana::slice<0>( _aosoa_y, "current_positions" );
auto x = sliceReferencePosition();
auto u = sliceDisplacement();
Kokkos::RangePolicy<execution_space> policy( 0, n_local + n_ghost );
Kokkos::RangePolicy<execution_space> policy( frozenOffset(),
referenceOffset() );
auto sum_x_u = KOKKOS_LAMBDA( const std::size_t pid )
{
for ( int d = 0; d < 3; d++ )
Expand All @@ -422,16 +428,17 @@ class Particles<MemorySpace, PMB, TemperatureIndependent, BaseOutput, Dimension>
void resize( int new_local, int new_ghost )
{
_timer.start();
n_local = new_local;
n_ghost = new_ghost;

_plist_x.aosoa().resize( new_local + new_ghost );
_aosoa_u.resize( new_local + new_ghost );
_aosoa_y.resize( new_local + new_ghost );
_aosoa_vol.resize( new_local + new_ghost );
_plist_f.aosoa().resize( new_local );
_aosoa_other.resize( new_local );
_aosoa_nofail.resize( new_local + new_ghost );
local_offset = new_local;
num_ghost = new_ghost;
size = new_local + new_ghost;

_plist_x.aosoa().resize( referenceOffset() );
_aosoa_u.resize( referenceOffset() );
_aosoa_y.resize( referenceOffset() );
_aosoa_vol.resize( referenceOffset() );
_plist_f.aosoa().resize( localOffset() );
_aosoa_other.resize( localOffset() );
_aosoa_nofail.resize( referenceOffset() );
size = _plist_x.size();
_timer.stop();
};
Expand All @@ -444,6 +451,7 @@ class Particles<MemorySpace, PMB, TemperatureIndependent, BaseOutput, Dimension>
return sliceCurrentPosition();
}

// TODO: enable ignoring frozen particles.
template <typename... OtherFields>
void output( [[maybe_unused]] const int output_step,
[[maybe_unused]] const double output_time,
Expand All @@ -455,15 +463,15 @@ class Particles<MemorySpace, PMB, TemperatureIndependent, BaseOutput, Dimension>
#ifdef Cabana_ENABLE_HDF5
Cabana::Experimental::HDF5ParticleOutput::writeTimeStep(
h5_config, "particles", MPI_COMM_WORLD, output_step, output_time,
n_local, getPosition( use_reference ), sliceForce(),
local_offset, getPosition( use_reference ), sliceForce(),
sliceDisplacement(), sliceVelocity(),
std::forward<OtherFields>( other )... );
#else
#ifdef Cabana_ENABLE_SILO
Cabana::Grid::Experimental::SiloParticleOutput::
writePartialRangeTimeStep(
"particles", local_grid->globalGrid(), output_step, output_time,
0, n_local, getPosition( use_reference ), sliceForce(),
0, local_offset, getPosition( use_reference ), sliceForce(),
sliceDisplacement(), sliceVelocity(),
std::forward<OtherFields>( other )... );

Expand Down Expand Up @@ -516,12 +524,6 @@ class Particles<MemorySpace, LPS, TemperatureIndependent, BaseOutput, Dimension>
using memory_space = typename base_type::memory_space;
using base_type::dim;

// Per particle.
using base_type::n_ghost;
using base_type::n_global;
using base_type::n_local;
using base_type::size;

// These are split since weighted volume only needs to be communicated once
// and dilatation only needs to be communicated for LPS.
using scalar_type = typename base_type::scalar_type;
Expand Down Expand Up @@ -565,8 +567,10 @@ class Particles<MemorySpace, LPS, TemperatureIndependent, BaseOutput, Dimension>
max_halo_width )
{
_init_timer.start();
_aosoa_m = aosoa_m_type( "Particle Weighted Volumes", n_local );
_aosoa_theta = aosoa_theta_type( "Particle Dilatations", n_local );
_aosoa_m = aosoa_m_type( "Particle Weighted Volumes",
base_type::localOffset() );
_aosoa_theta = aosoa_theta_type( "Particle Dilatations",
base_type::localOffset() );
init_lps();
_init_timer.stop();
}
Expand All @@ -577,8 +581,8 @@ class Particles<MemorySpace, LPS, TemperatureIndependent, BaseOutput, Dimension>
// Forward arguments to standard or custom particle creation.
base_type::createParticles( std::forward<Args>( args )... );
_init_timer.start();
_aosoa_m.resize( n_local );
_aosoa_theta.resize( n_local );
_aosoa_m.resize( base_type::localOffset() );
_aosoa_theta.resize( base_type::localOffset() );
_init_timer.stop();
}

Expand All @@ -603,8 +607,8 @@ class Particles<MemorySpace, LPS, TemperatureIndependent, BaseOutput, Dimension>
{
base_type::resize( new_local, new_ghost );
_timer.start();
_aosoa_theta.resize( new_local + new_ghost );
_aosoa_m.resize( new_local + new_ghost );
_aosoa_theta.resize( base_type::referenceOffset() );
_aosoa_m.resize( base_type::referenceOffset() );
_timer.stop();
}

Expand Down Expand Up @@ -651,12 +655,6 @@ class Particles<MemorySpace, PMB, TemperatureDependent, BaseOutput, Dimension>
using memory_space = typename base_type::memory_space;
using base_type::dim;

// Per particle.
using base_type::n_ghost;
using base_type::n_global;
using base_type::n_local;
using base_type::size;

// These are split since weighted volume only needs to be communicated once
// and dilatation only needs to be communicated for LPS.
using temp_types = Cabana::MemberTypes<double, double>;
Expand Down Expand Up @@ -695,7 +693,8 @@ class Particles<MemorySpace, PMB, TemperatureDependent, BaseOutput, Dimension>
: base_type( exec_space, low_corner, high_corner, num_cells,
max_halo_width )
{
_aosoa_temp = aosoa_temp_type( "Particle Temperature", n_local );
_aosoa_temp =
aosoa_temp_type( "Particle Temperature", base_type::localOffset() );
init_temp();
}

Expand All @@ -704,7 +703,7 @@ class Particles<MemorySpace, PMB, TemperatureDependent, BaseOutput, Dimension>
{
// Forward arguments to standard or custom particle creation.
base_type::createParticles( std::forward<Args>( args )... );
_aosoa_temp.resize( n_local );
_aosoa_temp.resize( base_type::localOffset() );
}

auto sliceTemperature()
Expand Down Expand Up @@ -735,7 +734,7 @@ class Particles<MemorySpace, PMB, TemperatureDependent, BaseOutput, Dimension>
void resize( int new_local, int new_ghost )
{
base_type::resize( new_local, new_ghost );
_aosoa_temp.resize( new_local + new_ghost );
_aosoa_temp.resize( base_type::referenceOffset() );
}

template <typename... OtherFields>
Expand Down Expand Up @@ -777,12 +776,6 @@ class Particles<MemorySpace, ModelType, ThermalType, EnergyOutput, Dimension>
using memory_space = typename base_type::memory_space;
using base_type::dim;

// Per particle.
using base_type::n_ghost;
using base_type::n_global;
using base_type::n_local;
using base_type::size;

// energy, damage
using output_types = Cabana::MemberTypes<double, double>;
using aosoa_output_type = Cabana::AoSoA<output_types, memory_space, 1>;
Expand Down Expand Up @@ -820,7 +813,8 @@ class Particles<MemorySpace, ModelType, ThermalType, EnergyOutput, Dimension>
: base_type( exec_space, low_corner, high_corner, num_cells,
max_halo_width )
{
_aosoa_output = aosoa_output_type( "Particle Output Fields", n_local );
_aosoa_output = aosoa_output_type( "Particle Output Fields",
base_type::localOffset() );
init_output();
}

Expand All @@ -829,7 +823,7 @@ class Particles<MemorySpace, ModelType, ThermalType, EnergyOutput, Dimension>
{
// Forward arguments to standard or custom particle creation.
base_type::createParticles( std::forward<Args>( args )... );
_aosoa_output.resize( n_local );
_aosoa_output.resize( base_type::localOffset() );
}

auto sliceStrainEnergy()
Expand All @@ -849,7 +843,7 @@ class Particles<MemorySpace, ModelType, ThermalType, EnergyOutput, Dimension>
void resize( int new_local, int new_ghost )
{
base_type::resize( new_local, new_ghost );
_aosoa_output.resize( new_local + new_ghost );
_aosoa_output.resize( base_type::localOffset() );
}

template <typename... OtherFields>
Expand Down
Loading

0 comments on commit 391444e

Please sign in to comment.