Prepare tagging 1.0.1 (#195)

* Lift GPU restrictions on using typical values for chemistry.

* Fix fuelName handling.

* Update release_notes in preparation for 1.0.1

* Skip ReSetTolODE on GPU. Used to set the global user_data used on CPU.

* Update submodules to match release notes.

* Start implementing ParallelFor(MF).

* Need to pass class indices into lambdas.

* Missing one.

RELEASE_NOTES.md

@@ -1,11 +1,22 @@
 # 1.0.1
-Release version 1.0.1 (In preparation)
+Release version 1.0.1 (Sept. 2021)
+### Major changes:
+   * EB new uses a 2nd order Godunov method with State Redistribution of the
+     expl. advection fluxes and Flux Redistribution of the expl. diffusion fluxes
+   * Advection functionalities now requires the AMReX-Hydro repository
 
 ### Minor changes
    * Revert some of HIP changes (AMREX_GPU_DEVICE -> AMREX_GPU_HOST_DEVICE)
    * Setup CMAKE compilation, AMReX/IAMR/PelePhysics submodules and automated testing (J. Rood)
    * Update EOS/Transport calls to match templated PelePhysics (L. Owen)
+   * Reduction of the number of fillPatch operation
+   * Species linear diffusion solve/apply are performed on species groups (default size NUM_SPECIES)
 
+### Dependencies stable \#
+   * PelePhysics : b2d45f0f68
+   * IAMR        : 80ca2cdc23
+   * AMReX       : a8fe43774d
+   * AMReX-Hydro : bea9f07d95
 
 # 1.0.0
 Release version 1.0.0 (January, 2021)

Source/PeleLM.cpp

@@ -1349,6 +1349,9 @@ PeleLM::init_mixture_fraction()
             Abort("Unknown mixtureFraction.format ! Should be 'Cantera' or 'RealList'");
          }
       }
+      if (fuelName.empty() && !hasUserMF) {
+          Print() << " Mixture fraction definition lacks fuelName: consider using ns.fuelName keyword \n";
+      }
 
       // Only interested in CHON -in that order. Compute Bilger weights
       amrex::Real atwCHON[4] = {0.0};
@@ -1493,8 +1496,7 @@ PeleLM::update_typical_values_chem ()
 {
 #ifdef USE_SUNDIALS_PP
   if (use_typ_vals_chem) {
-#ifndef AMREX_USE_GPU
-    if (verbose) amrex::Print() << "Using typical values for the absolute tolerances of the ode solver\n";
+    if (verbose>1) amrex::Print() << "Using typical values for the absolute tolerances of the ode solver\n";
 #ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
@@ -1508,12 +1510,10 @@ PeleLM::update_typical_values_chem ()
       }
       typical_values_chem[NUM_SPECIES] = typical_values[Temp];
       SetTypValsODE(typical_values_chem);
+#ifndef AMREX_USE_GPU
       ReSetTolODE();
-    }
-#else
-    // TODO: set this option back on in PP
-    amrex::Print() << "Using typical values for the absolute tolerances of the ode solver not available on GPU right now\n";
 #endif
+    }
   }
 #endif
 }
@@ -1952,24 +1952,16 @@ PeleLM::initData ()
     ProbParm const* lprobparm = prob_parm.get();
     PmfData const* lpmfdata = pmf_data_g;
 
-#ifdef AMREX_USE_OMP
-#pragma omp parallel if (Gpu::notInLaunchRegion())
-#endif
-    for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi)
+    auto const& sma = S_new.arrays();
+    amrex::ParallelFor(S_new,
+    [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept
     {
-      const Box& box = mfi.tilebox();
-      auto sfab = S_new.array(mfi);
-
-      amrex::ParallelFor(box,
-      [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
-      {
 #ifdef AMREX_USE_NEWMECH
-        amrex::Abort("USE_NEWMECH feature no longer working and has to be fixed/redone");
+       amrex::Abort("USE_NEWMECH feature no longer working and has to be fixed/redone");
 #else
-        pelelm_initdata(i, j, k, sfab, geomdata, *lprobparm, lpmfdata);
+       pelelm_initdata(i, j, k, sma[box_no], geomdata, *lprobparm, lpmfdata);
 #endif
-      });
-    }
+    });
   }
 
   showMFsub("1D",S_new,stripBox,"1D_S",level);
@@ -2152,24 +2144,19 @@ PeleLM::compute_instantaneous_reaction_rates (MultiFab&       R,
    maskMF.ParallelCopy(ebmask,0,0,1);
 #endif
 
-#ifdef AMREX_USE_OMP
-#pragma omp parallel if (Gpu::notInLaunchRegion())
-#endif
-   for (MFIter mfi(S,TilingIfNotGPU()); mfi.isValid(); ++mfi)
+   auto const& sma    = S.const_arrays();
+   auto const& maskma = maskMF.const_arrays();
+   auto const& rma    = R.arrays();
+   amrex::ParallelFor(S, [=,FS=first_spec,RH=RhoH,Temp=Temp]
+   AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept
    {
-      const Box& bx = mfi.tilebox();
-      auto const& rhoY    = S.const_array(mfi,first_spec);
-      auto const& rhoH    = S.const_array(mfi,RhoH);
-      auto const& T       = S.const_array(mfi,Temp);
-      auto const& mask    = maskMF.const_array(mfi);
-      auto const& rhoYdot = R.array(mfi);
-
-      amrex::ParallelFor(bx, [rhoY, rhoH, T, mask, rhoYdot]
-      AMREX_GPU_DEVICE (int i, int j, int k) noexcept
-      {
-         reactionRateRhoY( i, j, k, rhoY, rhoH, T, mask, rhoYdot );
-      });
-   }
+      reactionRateRhoY(i, j, k,
+                       Array4<Real const>(sma[box_no],FS),
+                       Array4<Real const>(sma[box_no],RH),
+                       Array4<Real const>(sma[box_no],Temp),
+                       maskma[box_no],
+                       rma[box_no]);
+   });
 
    if ((nGrow>0) && (how == HT_EXTRAP_GROW_CELLS))
    {
@@ -2210,25 +2197,18 @@ PeleLM::init (AmrLevel& old)
    FillPatchIterator FctCntfpi(*oldht,FuncCount,FuncCount.nGrow(),tnp1,FuncCount_Type,0,1);
    const MultiFab& FuncCount_old = FctCntfpi.get_mf();
 
-#ifdef AMREX_USE_OMP
-#pragma omp parallel if (Gpu::notInLaunchRegion())
-#endif
-   for (MFIter mfi(Ydot_old,TilingIfNotGPU()); mfi.isValid(); ++mfi)
+   auto const& orYdotma = Ydot_old.const_arrays();
+   auto const& nrYdotma = Ydot.arrays();
+   auto const& oFctCma  = FuncCount_old.const_arrays();
+   auto const& nFctCma  = FuncCount.arrays();
+   amrex::ParallelFor(Ydot_old,
+   [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept
    {
-      const Box& bx         = mfi.tilebox();
-      auto const& rhoYdot_n = Ydot.array(mfi);
-      auto const& rhoYdot_o = Ydot_old.const_array(mfi);
-      auto const& FctCnt_n  = FuncCount.array(mfi);
-      auto const& FctCnt_o  = FuncCount_old.const_array(mfi);
-      amrex::ParallelFor(bx, [rhoYdot_n, rhoYdot_o, FctCnt_n, FctCnt_o]
-      AMREX_GPU_DEVICE (int i, int j, int k) noexcept
-      {
-         for (int n = 0; n < NUM_SPECIES; n++) {
-            rhoYdot_n(i,j,k,n) = rhoYdot_o(i,j,k,n);
-         }
-         FctCnt_n(i,j,k) = FctCnt_o(i,j,k);
-      });
-   }
+      nFctCma[box_no](i,j,k) = oFctCma[box_no](i,j,k);
+      for (int n = 0; n < NUM_SPECIES; n++) {
+         nrYdotma[box_no](i,j,k,n) = orYdotma[box_no](i,j,k,n);
+      }
+   });
 }
 
 //
@@ -8374,23 +8354,16 @@ PeleLM::RhoH_to_Temp (MultiFab& S,
   AMREX_ALWAYS_ASSERT(nGrow <= S.nGrow());
 
   // TODO: simplified version of that function for now: no iters, no tols, ... PPhys need to be fixed
-
-#ifdef AMREX_USE_OMP
-#pragma omp parallel if (Gpu::notInLaunchRegion())
-#endif
-  for (MFIter mfi(S,TilingIfNotGPU()); mfi.isValid(); ++mfi)
+  auto const& sma    = S.arrays();
+  amrex::ParallelFor(S, [=,Dens=Density,FS=first_spec,RH=RhoH,Temp=Temp]
+  AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept
   {
-     const Box& bx    = mfi.tilebox();
-     auto const& T    = S.array(mfi,Temp);
-     auto const& rho  = S.const_array(mfi,Density);
-     auto const& rhoY = S.const_array(mfi,first_spec);
-     auto const& rhoH = S.const_array(mfi,RhoH);
-     amrex::ParallelFor(bx, [T,rho,rhoY,rhoH]
-     AMREX_GPU_DEVICE (int i, int j, int k) noexcept
-     {
-        getTfromHY(i,j,k, rho, rhoY, rhoH, T);
-     });
-  }
+     getTfromHY(i, j, k,
+                Array4<Real const>(sma[box_no],Dens),
+                Array4<Real const>(sma[box_no],FS),
+                Array4<Real const>(sma[box_no],RH),
+                Array4<Real>(sma[box_no],Temp));
+  });
 
   /*
   //
@@ -9346,6 +9319,9 @@ PeleLM::initActiveControl()
 
    if ( !ctrl_use_temp ) {
       // Get the fuel rhoY
+      if (fuelName.empty()) {
+          Abort("Using activeControl based on fuel mass requires ns.fuelName !");
+      }
       Vector<std::string> specNames;
       pele::physics::eos::speciesNames<pele::physics::PhysicsType::eos_type>(specNames);
       int fuelidx = -1;

Source/PeleLM_setup.cpp

@@ -341,9 +341,9 @@ set_species_bc (BCRec&       bc,
 //
 // Indices of fuel and oxidizer -- ParmParsed in & used in a couple places.
 //
-std::string PeleLM::fuelName        = "CH4";
+std::string PeleLM::fuelName        = "";
 std::string PeleLM::productName     = "CO2";
-Vector<std::string> PeleLM::consumptionName(1);
+Vector<std::string> PeleLM::consumptionName;
 static std::string oxidizerName     = "O2";
 
 //
@@ -376,15 +376,12 @@ PeleLM::variableSetUp ()
 
   amrex::Print() << " Initialization of reactor... \n";
   int reactor_type = 2;
-#ifdef _OPENMP
+#ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif  
 {
-// TODO: restore this option for GPU in PP
-#ifndef AMREX_USE_GPU
 #ifdef USE_SUNDIALS_PP
   SetTolFactODE(relative_tol_chem,absolute_tol_chem);
-#endif
 #endif
   reactor_init(reactor_type,ncells_chem);
 }
@@ -418,7 +415,10 @@ PeleLM::variableSetUp ()
   //
   ParmParse ppns("ns");
   ppns.query("fuelName",fuelName);
-  consumptionName[0] = fuelName;
+  if ( !fuelName.empty() ) {
+     consumptionName.resize(1);
+     consumptionName[0] = fuelName;
+  }
   if (int nc = ppns.countval("consumptionName"))
   {
     consumptionName.resize(nc);