Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix openPMD plotfile output #804

Merged
merged 7 commits into from
Dec 9, 2024
Merged

fix openPMD plotfile output #804

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f3e0b5e
add warning about particles
BenWibking Dec 2, 2024
c3407ac
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Dec 2, 2024
6c90800
fix average_fc_to_cc when mf_cc has no ghost cells
BenWibking Dec 2, 2024
258d15c
update container
BenWibking Dec 2, 2024
542a9f2
can't fill more ghosts than we have
BenWibking Dec 3, 2024
377fdab
fix number of ghosts
BenWibking Dec 3, 2024
e052beb
revert Dockerfile changes
BenWibking Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 14 additions & 13 deletions src/simulation.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,8 +216,7 @@ template <typename problem_t> class AMRSimulation : public amrex::AmrCore
void computeTimestep();
auto computeTimestepAtLevel(int lev) -> amrex::Real;

void AverageFCToCC(amrex::MultiFab &mf_cc, const amrex::MultiFab &mf_fc, int idim, int dstcomp_start, int srccomp_start, int srccomp_total,
int nGrow) const;
void AverageFCToCC(amrex::MultiFab &mf_cc, const amrex::MultiFab &mf_fc, int idim, int dstcomp_start, int srccomp_start, int srccomp_total) const;

virtual void computeMaxSignalLocal(int level) = 0;
virtual auto computeExtraPhysicsTimestep(int lev) -> amrex::Real = 0;
Expand Down Expand Up @@ -586,11 +585,9 @@ template <typename problem_t> void AMRSimulation<problem_t>::PerformanceHints()
}

#ifdef QUOKKA_USE_OPENPMD
// warning about face-centered variables and OpenPMD outputs
if constexpr (Physics_Indices<problem_t>::nvarTotal_fc > 0) {
amrex::Print() << "\n[Warning] [I/O] Plotfiles will ONLY contain cell-centered averages of face-centered variables!"
<< " Support for outputting face-centered variables for openPMD is not yet implemented.\n";
}
// warning about particles and OpenPMD outputs
amrex::Print() << "\n[Warning] [I/O] OpenPMD outputs currently do NOT include particles!"
<< " Support for outputting particles for openPMD is not yet implemented.\n";
#endif
}

Expand Down Expand Up @@ -2079,7 +2076,7 @@ template <typename problem_t> auto AMRSimulation<problem_t>::CustomPlotFileName(

template <typename problem_t>
void AMRSimulation<problem_t>::AverageFCToCC(amrex::MultiFab &mf_cc, const amrex::MultiFab &mf_fc, int idim, int dstcomp_start, int srccomp_start,
int srccomp_total, int nGrow) const
int srccomp_total) const
{
int di = 0;
int dj = 0;
Expand All @@ -2094,7 +2091,10 @@ void AMRSimulation<problem_t>::AverageFCToCC(amrex::MultiFab &mf_cc, const amrex
// iterate over the domain
auto const &state_cc = mf_cc.arrays();
markkrumholz marked this conversation as resolved.
Show resolved Hide resolved
auto const &state_fc = mf_fc.const_arrays();
amrex::ParallelFor(mf_cc, amrex::IntVect(AMREX_D_DECL(nGrow, nGrow, nGrow)), [=] AMREX_GPU_DEVICE(int boxidx, int i, int j, int k) {
int const ng_cc = mf_cc.nGrow();
int const ng_fc = mf_fc.nGrow();
AMREX_ALWAYS_ASSERT(ng_cc <= ng_fc); // if this is false, we can't fill the ghost cells!
amrex::ParallelFor(mf_cc, amrex::IntVect(AMREX_D_DECL(ng_cc, ng_cc, ng_cc)), [=] AMREX_GPU_DEVICE(int boxidx, int i, int j, int k) {
for (int icomp = 0; icomp < srccomp_total; ++icomp) {
state_cc[boxidx](i, j, k, dstcomp_start + icomp) =
0.5 * (state_fc[boxidx](i, j, k, srccomp_start + icomp) + state_fc[boxidx](i + di, j + dj, k + dk, srccomp_start + icomp));
Expand Down Expand Up @@ -2143,7 +2143,7 @@ template <typename problem_t> auto AMRSimulation<problem_t>::PlotFileMFAtLevel(c
// compute cell-center averaged face-centred data
if constexpr (Physics_Indices<problem_t>::nvarTotal_fc > 0) {
for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
AverageFCToCC(plotMF, state_new_fc_[lev][idim], idim, comp, 0, ncomp_per_dim_fc, nghost_fc);
AverageFCToCC(plotMF, state_new_fc_[lev][idim], idim, comp, 0, ncomp_per_dim_fc);
comp += ncomp_per_dim_fc;
}
}
Expand Down Expand Up @@ -2230,7 +2230,7 @@ template <typename problem_t> void AMRSimulation<problem_t>::doDiagnostics()
if (computeVars) {
for (int lev{0}; lev <= finestLevel(); ++lev) {
diagMFVec[lev] = std::make_unique<amrex::MultiFab>(grids[lev], dmap[lev], m_diagVars.size(), 1);
amrex::MultiFab const mf = PlotFileMFAtLevel(lev, nghost_cc_);
amrex::MultiFab const mf = PlotFileMFAtLevel(lev, std::min(nghost_cc_, nghost_fc_));
auto const varnames = GetPlotfileVarNames();

for (int v{0}; v < m_diagVars.size(); ++v) {
Expand Down Expand Up @@ -2291,7 +2291,8 @@ template <typename problem_t> void AMRSimulation<problem_t>::RenderAscent()
BL_PROFILE("AMRSimulation::RenderAscent()");

// combine multifabs
amrex::Vector<amrex::MultiFab> mf = PlotFileMF(nghost_cc_);
const int included_ghosts = std::min(nghost_cc_, nghost_fc_);
amrex::Vector<amrex::MultiFab> mf = PlotFileMF(included_ghosts);
amrex::Vector<const amrex::MultiFab *> mf_ptr = amrex::GetVecOfConstPtrs(mf);
amrex::Vector<std::string> varnames;
varnames.insert(varnames.end(), componentNames_cc_.begin(), componentNames_cc_.end());
Expand Down Expand Up @@ -2354,7 +2355,7 @@ template <typename problem_t> void AMRSimulation<problem_t>::WritePlotFile()
#ifdef QUOKKA_USE_OPENPMD
int included_ghosts = 0;
#else
int included_ghosts = nghost_cc_;
int included_ghosts = std::min(nghost_cc_, nghost_fc_);
#endif
amrex::Vector<amrex::MultiFab> mf = PlotFileMF(included_ghosts);
amrex::Vector<const amrex::MultiFab *> mf_ptr = amrex::GetVecOfConstPtrs(mf);
Expand Down
Loading