From f3e0b5e04aaf9e2fa50bb47ce66105a6a1a942e3 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Mon, 2 Dec 2024 17:14:13 +0000 Subject: [PATCH 1/7] add warning about particles --- src/simulation.hpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/simulation.hpp b/src/simulation.hpp index 3019bcbe9..c74c42a94 100644 --- a/src/simulation.hpp +++ b/src/simulation.hpp @@ -586,11 +586,9 @@ template void AMRSimulation::PerformanceHints() } #ifdef QUOKKA_USE_OPENPMD - // warning about face-centered variables and OpenPMD outputs - if constexpr (Physics_Indices::nvarTotal_fc > 0) { - amrex::Print() << "\n[Warning] [I/O] Plotfiles will ONLY contain cell-centered averages of face-centered variables!" - << " Support for outputting face-centered variables for openPMD is not yet implemented.\n"; - } + // warning about particles and OpenPMD outputs + amrex::Print() << "\n[Warning] [I/O] OpenPMD outputs currently do NOT include particles!" + << " Support for outputting particles for openPMD is not yet implemented.\n"; #endif } From c3407ac96d64707d81d7bd2cde1c2aa103df6a63 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 18:22:47 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/simulation.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation.hpp b/src/simulation.hpp index c74c42a94..154d3a787 100644 --- a/src/simulation.hpp +++ b/src/simulation.hpp @@ -588,7 +588,7 @@ template void AMRSimulation::PerformanceHints() #ifdef QUOKKA_USE_OPENPMD // warning about particles and OpenPMD outputs amrex::Print() << "\n[Warning] [I/O] OpenPMD outputs currently do NOT include particles!" - << " Support for outputting particles for openPMD is not yet implemented.\n"; + << " Support for outputting particles for openPMD is not yet implemented.\n"; #endif } From 6c908006ca8618a3256a216b004a723f8f8c07b6 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Mon, 2 Dec 2024 18:30:02 +0000 Subject: [PATCH 3/7] fix average_fc_to_cc when mf_cc has no ghost cells --- src/simulation.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/simulation.hpp b/src/simulation.hpp index 154d3a787..e3190dd8e 100644 --- a/src/simulation.hpp +++ b/src/simulation.hpp @@ -216,8 +216,7 @@ template class AMRSimulation : public amrex::AmrCore void computeTimestep(); auto computeTimestepAtLevel(int lev) -> amrex::Real; - void AverageFCToCC(amrex::MultiFab &mf_cc, const amrex::MultiFab &mf_fc, int idim, int dstcomp_start, int srccomp_start, int srccomp_total, - int nGrow) const; + void AverageFCToCC(amrex::MultiFab &mf_cc, const amrex::MultiFab &mf_fc, int idim, int dstcomp_start, int srccomp_start, int srccomp_total) const; virtual void computeMaxSignalLocal(int level) = 0; virtual auto computeExtraPhysicsTimestep(int lev) -> amrex::Real = 0; @@ -2077,7 +2076,7 @@ template auto AMRSimulation::CustomPlotFileName( template void AMRSimulation::AverageFCToCC(amrex::MultiFab &mf_cc, const amrex::MultiFab &mf_fc, int idim, int dstcomp_start, int srccomp_start, - int srccomp_total, int nGrow) const + int srccomp_total) const { int di = 0; int dj = 0; @@ -2092,7 +2091,8 @@ void AMRSimulation::AverageFCToCC(amrex::MultiFab &mf_cc, const amrex // iterate over the domain auto const &state_cc = mf_cc.arrays(); auto const &state_fc = mf_fc.const_arrays(); - amrex::ParallelFor(mf_cc, amrex::IntVect(AMREX_D_DECL(nGrow, nGrow, nGrow)), [=] AMREX_GPU_DEVICE(int boxidx, int i, int j, int k) { + int const ng = mf_cc.nGrow(); + amrex::ParallelFor(mf_cc, amrex::IntVect(AMREX_D_DECL(ng, ng, ng)), [=] AMREX_GPU_DEVICE(int boxidx, int i, int j, int k) { for (int icomp = 0; icomp < srccomp_total; ++icomp) { state_cc[boxidx](i, j, k, dstcomp_start + icomp) = 0.5 * (state_fc[boxidx](i, j, k, srccomp_start + icomp) + state_fc[boxidx](i + di, j + dj, k + dk, srccomp_start + icomp)); @@ -2141,7 +2141,7 @@ template auto AMRSimulation::PlotFileMFAtLevel(c // compute cell-center averaged face-centred data if constexpr (Physics_Indices::nvarTotal_fc > 0) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - AverageFCToCC(plotMF, state_new_fc_[lev][idim], idim, comp, 0, ncomp_per_dim_fc, nghost_fc); + AverageFCToCC(plotMF, state_new_fc_[lev][idim], idim, comp, 0, ncomp_per_dim_fc); comp += ncomp_per_dim_fc; } } From 258d15c4f96b69650ceff7228e19029320060373 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Mon, 2 Dec 2024 18:40:08 +0000 Subject: [PATCH 4/7] update container --- .devcontainer/cuda-container/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/cuda-container/Dockerfile b/.devcontainer/cuda-container/Dockerfile index 77807a46e..aaec7acbf 100644 --- a/.devcontainer/cuda-container/Dockerfile +++ b/.devcontainer/cuda-container/Dockerfile @@ -1,5 +1,5 @@ # Use the NVIDIA CUDA image as the base image -FROM nvcr.io/nvidia/nvhpc:24.9-devel-cuda12.6-ubuntu24.04 +FROM nvcr.io/nvidia/nvhpc:24.11-devel-cuda12.6-ubuntu24.04 # Set environment variables for NVIDIA ENV NVIDIA_VISIBLE_DEVICES=all @@ -9,7 +9,7 @@ ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility RUN apt-get update -qq && apt-get upgrade -y -qq && \ apt-get install -y -qq --no-install-recommends \ build-essential wget curl git ninja-build gcc g++ \ - python3-dev python3-numpy python3-matplotlib python3-pip \ + python3-dev python3-numpy python3-matplotlib python3-venv python3-pip \ libhdf5-dev && \ apt-get --yes -qq clean && \ rm -rf /var/lib/apt/lists/* From 542a9f266733a4049dbf15f5656ea8f0676296dd Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Tue, 3 Dec 2024 01:54:03 +0000 Subject: [PATCH 5/7] can't fill more ghosts than we have --- src/simulation.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/simulation.hpp b/src/simulation.hpp index e3190dd8e..da37a5323 100644 --- a/src/simulation.hpp +++ b/src/simulation.hpp @@ -2091,8 +2091,10 @@ void AMRSimulation::AverageFCToCC(amrex::MultiFab &mf_cc, const amrex // iterate over the domain auto const &state_cc = mf_cc.arrays(); auto const &state_fc = mf_fc.const_arrays(); - int const ng = mf_cc.nGrow(); - amrex::ParallelFor(mf_cc, amrex::IntVect(AMREX_D_DECL(ng, ng, ng)), [=] AMREX_GPU_DEVICE(int boxidx, int i, int j, int k) { + int const ng_cc = mf_cc.nGrow(); + int const ng_fc = mf_fc.nGrow(); + AMREX_ALWAYS_ASSERT(ng_cc <= ng_fc); // if this is false, we can't fill the ghost cells! + amrex::ParallelFor(mf_cc, amrex::IntVect(AMREX_D_DECL(ng_cc, ng_cc, ng_cc)), [=] AMREX_GPU_DEVICE(int boxidx, int i, int j, int k) { for (int icomp = 0; icomp < srccomp_total; ++icomp) { state_cc[boxidx](i, j, k, dstcomp_start + icomp) = 0.5 * (state_fc[boxidx](i, j, k, srccomp_start + icomp) + state_fc[boxidx](i + di, j + dj, k + dk, srccomp_start + icomp)); @@ -2352,7 +2354,7 @@ template void AMRSimulation::WritePlotFile() #ifdef QUOKKA_USE_OPENPMD int included_ghosts = 0; #else - int included_ghosts = nghost_cc_; + int included_ghosts = std::min(nghost_cc_, nghost_fc_); #endif amrex::Vector mf = PlotFileMF(included_ghosts); amrex::Vector mf_ptr = amrex::GetVecOfConstPtrs(mf); From 377fdab0099c2d96e594bbf14baf13bf9ef67673 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Tue, 3 Dec 2024 21:21:24 +0000 Subject: [PATCH 6/7] fix number of ghosts --- src/simulation.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/simulation.hpp b/src/simulation.hpp index da37a5323..486805ccb 100644 --- a/src/simulation.hpp +++ b/src/simulation.hpp @@ -2230,7 +2230,7 @@ template void AMRSimulation::doDiagnostics() if (computeVars) { for (int lev{0}; lev <= finestLevel(); ++lev) { diagMFVec[lev] = std::make_unique(grids[lev], dmap[lev], m_diagVars.size(), 1); - amrex::MultiFab const mf = PlotFileMFAtLevel(lev, nghost_cc_); + amrex::MultiFab const mf = PlotFileMFAtLevel(lev, std::min(nghost_cc_, nghost_fc_)); auto const varnames = GetPlotfileVarNames(); for (int v{0}; v < m_diagVars.size(); ++v) { @@ -2291,7 +2291,8 @@ template void AMRSimulation::RenderAscent() BL_PROFILE("AMRSimulation::RenderAscent()"); // combine multifabs - amrex::Vector mf = PlotFileMF(nghost_cc_); + const int included_ghosts = std::min(nghost_cc_, nghost_fc_); + amrex::Vector mf = PlotFileMF(included_ghosts); amrex::Vector mf_ptr = amrex::GetVecOfConstPtrs(mf); amrex::Vector varnames; varnames.insert(varnames.end(), componentNames_cc_.begin(), componentNames_cc_.end()); From e052beb5c19b3f025b1265913f37472ee4a77bd2 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Tue, 3 Dec 2024 21:23:44 +0000 Subject: [PATCH 7/7] revert Dockerfile changes --- .devcontainer/cuda-container/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/cuda-container/Dockerfile b/.devcontainer/cuda-container/Dockerfile index aaec7acbf..77807a46e 100644 --- a/.devcontainer/cuda-container/Dockerfile +++ b/.devcontainer/cuda-container/Dockerfile @@ -1,5 +1,5 @@ # Use the NVIDIA CUDA image as the base image -FROM nvcr.io/nvidia/nvhpc:24.11-devel-cuda12.6-ubuntu24.04 +FROM nvcr.io/nvidia/nvhpc:24.9-devel-cuda12.6-ubuntu24.04 # Set environment variables for NVIDIA ENV NVIDIA_VISIBLE_DEVICES=all @@ -9,7 +9,7 @@ ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility RUN apt-get update -qq && apt-get upgrade -y -qq && \ apt-get install -y -qq --no-install-recommends \ build-essential wget curl git ninja-build gcc g++ \ - python3-dev python3-numpy python3-matplotlib python3-venv python3-pip \ + python3-dev python3-numpy python3-matplotlib python3-pip \ libhdf5-dev && \ apt-get --yes -qq clean && \ rm -rf /var/lib/apt/lists/*