diff --git a/src/simulation.hpp b/src/simulation.hpp index 3019bcbe9..486805ccb 100644 --- a/src/simulation.hpp +++ b/src/simulation.hpp @@ -216,8 +216,7 @@ template class AMRSimulation : public amrex::AmrCore void computeTimestep(); auto computeTimestepAtLevel(int lev) -> amrex::Real; - void AverageFCToCC(amrex::MultiFab &mf_cc, const amrex::MultiFab &mf_fc, int idim, int dstcomp_start, int srccomp_start, int srccomp_total, - int nGrow) const; + void AverageFCToCC(amrex::MultiFab &mf_cc, const amrex::MultiFab &mf_fc, int idim, int dstcomp_start, int srccomp_start, int srccomp_total) const; virtual void computeMaxSignalLocal(int level) = 0; virtual auto computeExtraPhysicsTimestep(int lev) -> amrex::Real = 0; @@ -586,11 +585,9 @@ template void AMRSimulation::PerformanceHints() } #ifdef QUOKKA_USE_OPENPMD - // warning about face-centered variables and OpenPMD outputs - if constexpr (Physics_Indices::nvarTotal_fc > 0) { - amrex::Print() << "\n[Warning] [I/O] Plotfiles will ONLY contain cell-centered averages of face-centered variables!" - << " Support for outputting face-centered variables for openPMD is not yet implemented.\n"; - } + // warning about particles and OpenPMD outputs + amrex::Print() << "\n[Warning] [I/O] OpenPMD outputs currently do NOT include particles!" + << " Support for outputting particles for openPMD is not yet implemented.\n"; #endif } @@ -2079,7 +2076,7 @@ template auto AMRSimulation::CustomPlotFileName( template void AMRSimulation::AverageFCToCC(amrex::MultiFab &mf_cc, const amrex::MultiFab &mf_fc, int idim, int dstcomp_start, int srccomp_start, - int srccomp_total, int nGrow) const + int srccomp_total) const { int di = 0; int dj = 0; @@ -2094,7 +2091,10 @@ void AMRSimulation::AverageFCToCC(amrex::MultiFab &mf_cc, const amrex // iterate over the domain auto const &state_cc = mf_cc.arrays(); auto const &state_fc = mf_fc.const_arrays(); - amrex::ParallelFor(mf_cc, amrex::IntVect(AMREX_D_DECL(nGrow, nGrow, nGrow)), [=] AMREX_GPU_DEVICE(int boxidx, int i, int j, int k) { + int const ng_cc = mf_cc.nGrow(); + int const ng_fc = mf_fc.nGrow(); + AMREX_ALWAYS_ASSERT(ng_cc <= ng_fc); // if this is false, we can't fill the ghost cells! + amrex::ParallelFor(mf_cc, amrex::IntVect(AMREX_D_DECL(ng_cc, ng_cc, ng_cc)), [=] AMREX_GPU_DEVICE(int boxidx, int i, int j, int k) { for (int icomp = 0; icomp < srccomp_total; ++icomp) { state_cc[boxidx](i, j, k, dstcomp_start + icomp) = 0.5 * (state_fc[boxidx](i, j, k, srccomp_start + icomp) + state_fc[boxidx](i + di, j + dj, k + dk, srccomp_start + icomp)); @@ -2143,7 +2143,7 @@ template auto AMRSimulation::PlotFileMFAtLevel(c // compute cell-center averaged face-centred data if constexpr (Physics_Indices::nvarTotal_fc > 0) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - AverageFCToCC(plotMF, state_new_fc_[lev][idim], idim, comp, 0, ncomp_per_dim_fc, nghost_fc); + AverageFCToCC(plotMF, state_new_fc_[lev][idim], idim, comp, 0, ncomp_per_dim_fc); comp += ncomp_per_dim_fc; } } @@ -2230,7 +2230,7 @@ template void AMRSimulation::doDiagnostics() if (computeVars) { for (int lev{0}; lev <= finestLevel(); ++lev) { diagMFVec[lev] = std::make_unique(grids[lev], dmap[lev], m_diagVars.size(), 1); - amrex::MultiFab const mf = PlotFileMFAtLevel(lev, nghost_cc_); + amrex::MultiFab const mf = PlotFileMFAtLevel(lev, std::min(nghost_cc_, nghost_fc_)); auto const varnames = GetPlotfileVarNames(); for (int v{0}; v < m_diagVars.size(); ++v) { @@ -2291,7 +2291,8 @@ template void AMRSimulation::RenderAscent() BL_PROFILE("AMRSimulation::RenderAscent()"); // combine multifabs - amrex::Vector mf = PlotFileMF(nghost_cc_); + const int included_ghosts = std::min(nghost_cc_, nghost_fc_); + amrex::Vector mf = PlotFileMF(included_ghosts); amrex::Vector mf_ptr = amrex::GetVecOfConstPtrs(mf); amrex::Vector varnames; varnames.insert(varnames.end(), componentNames_cc_.begin(), componentNames_cc_.end()); @@ -2354,7 +2355,7 @@ template void AMRSimulation::WritePlotFile() #ifdef QUOKKA_USE_OPENPMD int included_ghosts = 0; #else - int included_ghosts = nghost_cc_; + int included_ghosts = std::min(nghost_cc_, nghost_fc_); #endif amrex::Vector mf = PlotFileMF(included_ghosts); amrex::Vector mf_ptr = amrex::GetVecOfConstPtrs(mf);