From 6add625ec96275f783d056b6c8daa0e1ff8e5f81 Mon Sep 17 00:00:00 2001 From: Lucas Esclapez <13371051+esclapez@users.noreply.github.com> Date: Tue, 7 Sep 2021 22:29:13 -0700 Subject: [PATCH] Prepare tagging 1.0.1 (#195) * Lift GPU restrictions on using typical values for chemistry. * Fix fuelName handling. * Update release_notes in preparation for 1.0.1 * Skip ReSetTolODE on GPU. Used to set the global user_data used on CPU. * Update submodules to match release notes. * Start implementing ParallelFor(MF). * Need to pass class indices into lambdas. * Missing one. --- RELEASE_NOTES.md | 13 ++++- Source/PeleLM.cpp | 118 ++++++++++++++++------------------------ Source/PeleLM_setup.cpp | 14 ++--- Submodules/AMReX-Hydro | 2 +- Submodules/IAMR | 2 +- Submodules/PelePhysics | 2 +- Submodules/amrex | 2 +- 7 files changed, 70 insertions(+), 83 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index eb88d46a..c04780f1 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,11 +1,22 @@ # 1.0.1 -Release version 1.0.1 (In preparation) +Release version 1.0.1 (Sept. 2021) +### Major changes: + * EB new uses a 2nd order Godunov method with State Redistribution of the + expl. advection fluxes and Flux Redistribution of the expl. diffusion fluxes + * Advection functionalities now requires the AMReX-Hydro repository ### Minor changes * Revert some of HIP changes (AMREX_GPU_DEVICE -> AMREX_GPU_HOST_DEVICE) * Setup CMAKE compilation, AMReX/IAMR/PelePhysics submodules and automated testing (J. Rood) * Update EOS/Transport calls to match templated PelePhysics (L. Owen) + * Reduction of the number of fillPatch operation + * Species linear diffusion solve/apply are performed on species groups (default size NUM_SPECIES) +### Dependencies stable \# + * PelePhysics : b2d45f0f68 + * IAMR : 80ca2cdc23 + * AMReX : a8fe43774d + * AMReX-Hydro : bea9f07d95 # 1.0.0 Release version 1.0.0 (January, 2021) diff --git a/Source/PeleLM.cpp b/Source/PeleLM.cpp index a486e8ce..a91844a7 100644 --- a/Source/PeleLM.cpp +++ b/Source/PeleLM.cpp @@ -1349,6 +1349,9 @@ PeleLM::init_mixture_fraction() Abort("Unknown mixtureFraction.format ! Should be 'Cantera' or 'RealList'"); } } + if (fuelName.empty() && !hasUserMF) { + Print() << " Mixture fraction definition lacks fuelName: consider using ns.fuelName keyword \n"; + } // Only interested in CHON -in that order. Compute Bilger weights amrex::Real atwCHON[4] = {0.0}; @@ -1493,8 +1496,7 @@ PeleLM::update_typical_values_chem () { #ifdef USE_SUNDIALS_PP if (use_typ_vals_chem) { -#ifndef AMREX_USE_GPU - if (verbose) amrex::Print() << "Using typical values for the absolute tolerances of the ode solver\n"; + if (verbose>1) amrex::Print() << "Using typical values for the absolute tolerances of the ode solver\n"; #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif @@ -1508,12 +1510,10 @@ PeleLM::update_typical_values_chem () } typical_values_chem[NUM_SPECIES] = typical_values[Temp]; SetTypValsODE(typical_values_chem); +#ifndef AMREX_USE_GPU ReSetTolODE(); - } -#else - // TODO: set this option back on in PP - amrex::Print() << "Using typical values for the absolute tolerances of the ode solver not available on GPU right now\n"; #endif + } } #endif } @@ -1952,24 +1952,16 @@ PeleLM::initData () ProbParm const* lprobparm = prob_parm.get(); PmfData const* lpmfdata = pmf_data_g; -#ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) + auto const& sma = S_new.arrays(); + amrex::ParallelFor(S_new, + [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept { - const Box& box = mfi.tilebox(); - auto sfab = S_new.array(mfi); - - amrex::ParallelFor(box, - [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { #ifdef AMREX_USE_NEWMECH - amrex::Abort("USE_NEWMECH feature no longer working and has to be fixed/redone"); + amrex::Abort("USE_NEWMECH feature no longer working and has to be fixed/redone"); #else - pelelm_initdata(i, j, k, sfab, geomdata, *lprobparm, lpmfdata); + pelelm_initdata(i, j, k, sma[box_no], geomdata, *lprobparm, lpmfdata); #endif - }); - } + }); } showMFsub("1D",S_new,stripBox,"1D_S",level); @@ -2152,24 +2144,19 @@ PeleLM::compute_instantaneous_reaction_rates (MultiFab& R, maskMF.ParallelCopy(ebmask,0,0,1); #endif -#ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - for (MFIter mfi(S,TilingIfNotGPU()); mfi.isValid(); ++mfi) + auto const& sma = S.const_arrays(); + auto const& maskma = maskMF.const_arrays(); + auto const& rma = R.arrays(); + amrex::ParallelFor(S, [=,FS=first_spec,RH=RhoH,Temp=Temp] + AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept { - const Box& bx = mfi.tilebox(); - auto const& rhoY = S.const_array(mfi,first_spec); - auto const& rhoH = S.const_array(mfi,RhoH); - auto const& T = S.const_array(mfi,Temp); - auto const& mask = maskMF.const_array(mfi); - auto const& rhoYdot = R.array(mfi); - - amrex::ParallelFor(bx, [rhoY, rhoH, T, mask, rhoYdot] - AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - reactionRateRhoY( i, j, k, rhoY, rhoH, T, mask, rhoYdot ); - }); - } + reactionRateRhoY(i, j, k, + Array4(sma[box_no],FS), + Array4(sma[box_no],RH), + Array4(sma[box_no],Temp), + maskma[box_no], + rma[box_no]); + }); if ((nGrow>0) && (how == HT_EXTRAP_GROW_CELLS)) { @@ -2210,25 +2197,18 @@ PeleLM::init (AmrLevel& old) FillPatchIterator FctCntfpi(*oldht,FuncCount,FuncCount.nGrow(),tnp1,FuncCount_Type,0,1); const MultiFab& FuncCount_old = FctCntfpi.get_mf(); -#ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - for (MFIter mfi(Ydot_old,TilingIfNotGPU()); mfi.isValid(); ++mfi) + auto const& orYdotma = Ydot_old.const_arrays(); + auto const& nrYdotma = Ydot.arrays(); + auto const& oFctCma = FuncCount_old.const_arrays(); + auto const& nFctCma = FuncCount.arrays(); + amrex::ParallelFor(Ydot_old, + [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept { - const Box& bx = mfi.tilebox(); - auto const& rhoYdot_n = Ydot.array(mfi); - auto const& rhoYdot_o = Ydot_old.const_array(mfi); - auto const& FctCnt_n = FuncCount.array(mfi); - auto const& FctCnt_o = FuncCount_old.const_array(mfi); - amrex::ParallelFor(bx, [rhoYdot_n, rhoYdot_o, FctCnt_n, FctCnt_o] - AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - for (int n = 0; n < NUM_SPECIES; n++) { - rhoYdot_n(i,j,k,n) = rhoYdot_o(i,j,k,n); - } - FctCnt_n(i,j,k) = FctCnt_o(i,j,k); - }); - } + nFctCma[box_no](i,j,k) = oFctCma[box_no](i,j,k); + for (int n = 0; n < NUM_SPECIES; n++) { + nrYdotma[box_no](i,j,k,n) = orYdotma[box_no](i,j,k,n); + } + }); } // @@ -8374,23 +8354,16 @@ PeleLM::RhoH_to_Temp (MultiFab& S, AMREX_ALWAYS_ASSERT(nGrow <= S.nGrow()); // TODO: simplified version of that function for now: no iters, no tols, ... PPhys need to be fixed - -#ifdef AMREX_USE_OMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - for (MFIter mfi(S,TilingIfNotGPU()); mfi.isValid(); ++mfi) + auto const& sma = S.arrays(); + amrex::ParallelFor(S, [=,Dens=Density,FS=first_spec,RH=RhoH,Temp=Temp] + AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept { - const Box& bx = mfi.tilebox(); - auto const& T = S.array(mfi,Temp); - auto const& rho = S.const_array(mfi,Density); - auto const& rhoY = S.const_array(mfi,first_spec); - auto const& rhoH = S.const_array(mfi,RhoH); - amrex::ParallelFor(bx, [T,rho,rhoY,rhoH] - AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - getTfromHY(i,j,k, rho, rhoY, rhoH, T); - }); - } + getTfromHY(i, j, k, + Array4(sma[box_no],Dens), + Array4(sma[box_no],FS), + Array4(sma[box_no],RH), + Array4(sma[box_no],Temp)); + }); /* // @@ -9346,6 +9319,9 @@ PeleLM::initActiveControl() if ( !ctrl_use_temp ) { // Get the fuel rhoY + if (fuelName.empty()) { + Abort("Using activeControl based on fuel mass requires ns.fuelName !"); + } Vector specNames; pele::physics::eos::speciesNames(specNames); int fuelidx = -1; diff --git a/Source/PeleLM_setup.cpp b/Source/PeleLM_setup.cpp index 3ed514c0..266d55e3 100644 --- a/Source/PeleLM_setup.cpp +++ b/Source/PeleLM_setup.cpp @@ -341,9 +341,9 @@ set_species_bc (BCRec& bc, // // Indices of fuel and oxidizer -- ParmParsed in & used in a couple places. // -std::string PeleLM::fuelName = "CH4"; +std::string PeleLM::fuelName = ""; std::string PeleLM::productName = "CO2"; -Vector PeleLM::consumptionName(1); +Vector PeleLM::consumptionName; static std::string oxidizerName = "O2"; // @@ -376,15 +376,12 @@ PeleLM::variableSetUp () amrex::Print() << " Initialization of reactor... \n"; int reactor_type = 2; -#ifdef _OPENMP +#ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif { -// TODO: restore this option for GPU in PP -#ifndef AMREX_USE_GPU #ifdef USE_SUNDIALS_PP SetTolFactODE(relative_tol_chem,absolute_tol_chem); -#endif #endif reactor_init(reactor_type,ncells_chem); } @@ -418,7 +415,10 @@ PeleLM::variableSetUp () // ParmParse ppns("ns"); ppns.query("fuelName",fuelName); - consumptionName[0] = fuelName; + if ( !fuelName.empty() ) { + consumptionName.resize(1); + consumptionName[0] = fuelName; + } if (int nc = ppns.countval("consumptionName")) { consumptionName.resize(nc); diff --git a/Submodules/AMReX-Hydro b/Submodules/AMReX-Hydro index ca9ad099..bea9f07d 160000 --- a/Submodules/AMReX-Hydro +++ b/Submodules/AMReX-Hydro @@ -1 +1 @@ -Subproject commit ca9ad0992f48bbdc9030da8f01839d5875dbad9d +Subproject commit bea9f07d955892e5aa66b873e2028e699b2baf37 diff --git a/Submodules/IAMR b/Submodules/IAMR index df8b261c..80ca2cdc 160000 --- a/Submodules/IAMR +++ b/Submodules/IAMR @@ -1 +1 @@ -Subproject commit df8b261c9b12febfe3645a90b6953c9adbfbfba8 +Subproject commit 80ca2cdc233706d24673835e9b9301863f03bed1 diff --git a/Submodules/PelePhysics b/Submodules/PelePhysics index 55ca91b2..b2d45f0f 160000 --- a/Submodules/PelePhysics +++ b/Submodules/PelePhysics @@ -1 +1 @@ -Subproject commit 55ca91b221532e00aaf67eb8a5fd2a08f7bb1c4f +Subproject commit b2d45f0f683da5e3b5f89440ab35bf0e0df69c04 diff --git a/Submodules/amrex b/Submodules/amrex index d0b275b7..a8fe4377 160000 --- a/Submodules/amrex +++ b/Submodules/amrex @@ -1 +1 @@ -Subproject commit d0b275b7064e20b87272eda07ea9adef0f63cf2a +Subproject commit a8fe43774d8ad7437ff71bb253175d64a168d933