remove USE_POISSON flag and create anelastic CI test (#1936)

erf-model · Nov 7, 2024 · 91dcdd7 · 91dcdd7
1 parent f9111c3
commit 91dcdd7
Show file tree

Hide file tree

Showing 32 changed files with 2,657 additions and 136 deletions.
diff --git a/CMake/BuildERFExe.cmake b/CMake/BuildERFExe.cmake
@@ -25,13 +25,6 @@ function(build_erf_lib erf_lib_name)
     target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_WARM_NO_PRECIP)
   endif()
 
-  if(ERF_ENABLE_POISSON_SOLVE)
-    target_sources(${erf_lib_name} PRIVATE
-                   ${SRC_DIR}/Utils/ERF_PoissonSolve.cpp
-                   ${SRC_DIR}/Utils/ERF_PoissonSolve_tb.cpp)
-    target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_POISSON_SOLVE)
-  endif()
-
   if(ERF_ENABLE_PARTICLES)
     target_sources(${erf_lib_name} PRIVATE
                    ${SRC_DIR}/Particles/ERFPCEvolve.cpp
@@ -190,6 +183,8 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/Utils/ERF_AverageDown.cpp
        ${SRC_DIR}/Utils/ERF_ChopGrids.cpp
        ${SRC_DIR}/Utils/ERF_MomentumToVelocity.cpp
+       ${SRC_DIR}/Utils/ERF_PoissonSolve.cpp
+       ${SRC_DIR}/Utils/ERF_PoissonSolve_tb.cpp
        ${SRC_DIR}/Utils/ERF_TerrainMetrics.cpp
        ${SRC_DIR}/Utils/ERF_VelocityToMomentum.cpp
        ${SRC_DIR}/Utils/ERF_InteriorGhostCells.cpp

diff --git a/CMake/SetAmrexOptions.cmake b/CMake/SetAmrexOptions.cmake
@@ -27,12 +27,9 @@ set(AMReX_ACC OFF)
 set(AMReX_PLOTFILE_TOOLS ${ERF_ENABLE_FCOMPARE})
 set(AMReX_FORTRAN OFF)
 
-set(AMReX_LINEAR_SOLVERS OFF)
-if(ERF_ENABLE_POISSON_SOLVE)
-  set(AMReX_LINEAR_SOLVERS ON)
-  set(AMReX_LINEAR_SOLVERS_EM OFF)
-  set(AMReX_LINEAR_SOLVERS_INCFLO OFF)
-endif()
+set(AMReX_LINEAR_SOLVERS ON)
+set(AMReX_LINEAR_SOLVERS_EM OFF)
+set(AMReX_LINEAR_SOLVERS_INCFLO OFF)
 
 set(AMReX_PARTICLES OFF)
 if(ERF_ENABLE_PARTICLES)

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -24,8 +24,6 @@ option(ERF_ENABLE_MOISTURE "Enable Full Moisture" ON)
 option(ERF_ENABLE_WARM_NO_PRECIP "Enable Warm Moisture" OFF)
 option(ERF_ENABLE_RRTMGP "Enable RTE-RRTMGP Radiation" OFF)
 
-option(ERF_ENABLE_POISSON_SOLVE "Enable Poisson solve for anelastic/incompressible flow" OFF)
-
 #Options for performance
 option(ERF_ENABLE_MPI "Enable MPI" OFF)
 option(ERF_ENABLE_OPENMP "Enable OpenMP" OFF)

diff --git a/Docs/sphinx_doc/Inputs.rst b/Docs/sphinx_doc/Inputs.rst
@@ -34,10 +34,8 @@ Governing Equations
 
 .. note::
 
-   To solve the anelastic equations, you must set USE_POISSON_SOLVE = TRUE if using
-   gmake or ERF_ENABLE_POISSON_SOLVE if using cmake.  This will enable use of the
-   AMReX-based Poisson solver.   To optionally use the FFT solver, you must additionally
-   set USE_FFT = TRUE if using gmake.
+   To use the FFT solver if running with anelastic, you must set USE_FFT = TRUE if using gmake
+   or ERF_ENABLE_FFT if using cmake.
 
 Problem Geometry
 ================
@@ -1162,9 +1160,7 @@ This file has the same format as used by ``ideal.exe`` executable in WRF.
 Using this option for initialization, running ``ideal.exe`` and reading from the resulting ``wrfinput_d01`` file are not needed.
 This option is used for initializing ERF domain to a horizontally homogeneous mesoscale state and does not include terrain or map scale factors.
 
-In addition, there is a run-time option to project the initial velocity field to make it divergence-free.  To take
-advantage of this option, the code must be built with ``USE_POISSON_SOLVE = TRUE`` in the GNUmakefile if using gmake, or with
-``-DERF_ENABLE_POISSON_SOLVE:BOOL=ON`` in the cmake.sh file if using cmake.
+In addition, there is a run-time option to project the initial velocity field to make it divergence-free.
 
 List of Parameters
 ------------------
@@ -1231,9 +1227,6 @@ integrating the hydrostatic equation from the surface.
 
 If **erf.init_type = custom** or **erf.init_type = input_sounding**, ``erf.nc_init_file`` and ``erf.nc_bdy_file`` do not need to be set.
 
-Setting **erf.project_initial_velocity = 1** will have no effect if the code is not built with **ERF_USE_POISSON_SOLVE** defined
-if using cmake or with **USE_POISSON_SOLVE = TRUE** if using gmake.
-
 Map Scale Factors
 =================
 

diff --git a/Docs/sphinx_doc/building.rst b/Docs/sphinx_doc/building.rst
@@ -96,8 +96,6 @@ or if using tcsh,
    +--------------------+------------------------------+------------------+-------------+
    | USE_PARTICLES      | Whether to enable particles  | TRUE / FALSE     | FALSE       |
    +--------------------+------------------------------+------------------+-------------+
-   | USE_POISSON_SOLVE  | Whether to enable anelastic  | TRUE / FALSE     | FALSE       |
-   +--------------------+------------------------------+------------------+-------------+
    | USE_WARM_NO_PRECIP | Whether to use warm moisture | TRUE / FALSE     | FALSE       |
    +--------------------+------------------------------+------------------+-------------+
    | USE_MULTIBLOCK     | Whether to enable multiblock | TRUE / FALSE     | FALSE       |
@@ -113,9 +111,6 @@ or if using tcsh,
    | TRACE_PROFILE      | Include trace profiling info | TRUE / FALSE     | FALSE       |
    +--------------------+------------------------------+------------------+-------------+
 
-   .. note::
-      **To run with the anelastic option, USE_POISSON_SOLVE must be set to TRUE.**
-
    .. note::
       **At most one of USE_OMP, USE_CUDA, USE_HIP, USE_SYCL should be set to true.**
 
@@ -192,8 +187,6 @@ Analogous to GNU Make, the list of cmake directives is as follows:
    +---------------------------+------------------------------+------------------+-------------+
    | ERF_ENABLE_PARTICLES      | Whether to enable particles  | TRUE / FALSE     | FALSE       |
    +---------------------------+------------------------------+------------------+-------------+
-   | ERF_ENABLE_POISSON_SOVLE  | Whether to enable anelastic  | TRUE / FALSE     | FALSE       |
-   +---------------------------+------------------------------+------------------+-------------+
    | ERF_ENABLE_WARM_NO_PRECIP | Whether to use warm moisture | TRUE / FALSE     | FALSE       |
    +---------------------------+------------------------------+------------------+-------------+
    | ERF_ENABLE_MULTIBLOCK     | Whether to enable multiblock | TRUE / FALSE     | FALSE       |
@@ -205,9 +198,6 @@ Analogous to GNU Make, the list of cmake directives is as follows:
    | ERF_ENABLE_FCOMPARE       | Whether to enable fcompare   | TRUE / FALSE     | FALSE       |
    +---------------------------+------------------------------+------------------+-------------+
 
-   .. note::
-      **To run with the anelastic option, ERF_ENABLE_POISSON_SOLVE must be set to TRUE.**
-
    .. note::
       **At most one of ERF_ENABLE_OMP, ERF_ENABLE_CUDA, ERF_ENABLE_HIP and ERF_ENABLE_SYCL should be set to true.**
 

diff --git a/Exec/ABL/GNUmakefile b/Exec/ABL/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/ABL_input_sounding/GNUmakefile b/Exec/ABL_input_sounding/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-USE_POISSON_SOLVE = FALSE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/DevTests/FlowInABox/GNUmakefile b/Exec/DevTests/FlowInABox/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/DevTests/TemperatureSource/GNUmakefile b/Exec/DevTests/TemperatureSource/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/DryRegTests/DensityCurrent/GNUmakefile b/Exec/DryRegTests/DensityCurrent/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/DryRegTests/TurbulentInflow/GNUmakefile b/Exec/DryRegTests/TurbulentInflow/GNUmakefile
@@ -24,9 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-# Incompressible/Anelastic Solver
-USE_POISSON_SOLVE = FALSE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/Make.ERF b/Exec/Make.ERF
@@ -111,11 +111,9 @@ ifeq ($(USE_EB),TRUE)
 AMReXdirs             += EB
 endif
 
-ifeq ($(USE_POISSON_SOLVE),TRUE)
 USE_LINEAR_SOLVERS_INCFLO = FALSE
 USE_LINEAR_SOLVERS_EM     = FALSE
 AMReXdirs                += LinearSolvers
-endif
 
 ifeq ($(USE_HDF5),TRUE)
 AMReXdirs             += Extern/HDF5
@@ -234,11 +232,6 @@ ifeq ($(COMPUTE_ERROR), TRUE)
   DEFINES += -DERF_COMPUTE_ERROR
 endif
 
-ifeq ($(USE_POISSON_SOLVE), TRUE)
-  DEFINES += -DERF_USE_POISSON_SOLVE
-  USERSuffix += .INC
-endif
-
 ifeq ($(USE_PARTICLES), TRUE)
   DEFINES += -DERF_USE_PARTICLES
 endif

diff --git a/Exec/MoistRegTests/Bomex/GNUmakefile b/Exec/MoistRegTests/Bomex/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/WindFarmTests/AWAKEN/GNUmakefile b/Exec/WindFarmTests/AWAKEN/GNUmakefile
@@ -26,9 +26,6 @@ USE_ASSERTION = TRUE
 
 USE_WINDFARM = TRUE
 
-
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/WindFarmTests/EWP/GNUmakefile b/Exec/WindFarmTests/EWP/GNUmakefile
@@ -26,9 +26,6 @@ USE_ASSERTION = TRUE
 
 USE_WINDFARM = TRUE
 
-
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/WindFarmTests/SimpleActuatorDisk/GNUmakefile b/Exec/WindFarmTests/SimpleActuatorDisk/GNUmakefile
@@ -26,9 +26,6 @@ USE_ASSERTION = TRUE
 
 USE_WINDFARM = TRUE
 
-
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Source/DataStructs/ERF_DataStruct.H b/Source/DataStructs/ERF_DataStruct.H
@@ -198,7 +198,6 @@ struct SolverChoice {
 
         // *******************************************************************************
 
-#ifdef ERF_USE_POISSON_SOLVE
         bool any_anelastic = false;
         for (int i = 0; i <= max_level; ++i) {
             if (anelastic[i] == 1) any_anelastic = true;
@@ -225,14 +224,6 @@ struct SolverChoice {
                 substepping_type[lev] = SubsteppingType::None;
             }
         }
-#else
-        for (int i = 0; i <= max_level; ++i) {
-            if (anelastic[i] != 0) amrex::Abort("To run with anelastic you must compile with USE_POISSON_SOLVE = TRUE");
-        }
-        if (project_initial_velocity) {
-            amrex::Abort("To project the initial velocity you must compile with USE_POISSON_SOLVE = TRUE");
-        }
-#endif
 
         pp.query("force_stage1_single_substep", force_stage1_single_substep);
 

diff --git a/Source/ERF.cpp b/Source/ERF.cpp
@@ -856,7 +856,6 @@ ERF::InitData_post ()
         }
     }
 
-#ifdef ERF_USE_POISSON_SOLVE
     if (restart_chkfile == "")
     {
         // Note -- this projection is only defined for no terrain
@@ -870,7 +869,6 @@ ERF::InitData_post ()
             }
         }
     }
-#endif
 
     // Copy from new into old just in case
     for (int lev = 0; lev <= finest_level; ++lev)

diff --git a/Source/TimeIntegration/ERF_TI_no_substep_fun.H b/Source/TimeIntegration/ERF_TI_no_substep_fun.H
@@ -134,7 +134,6 @@
         apply_bcs(S_sum, time_for_fp, S_sum[IntVars::cons].nGrow(), S_sum[IntVars::xmom].nGrow(),
                   fast_only=true, vel_and_mom_synced=false);
 
-#ifdef ERF_USE_POISSON_SOLVE
         if (solverChoice.anelastic[level]) {
             bool have_tb = (thin_xforce[0] || thin_yforce[0] || thin_zforce[0]);
             if (solverChoice.project_every_stage || (nrk==1)) {
@@ -145,5 +144,4 @@
                 }
             }
         }
-#endif
     };
diff --git a/Source/TimeIntegration/ERF_TI_slow_headers.H b/Source/TimeIntegration/ERF_TI_slow_headers.H
@@ -98,9 +98,7 @@ void erf_slow_rhs_pre (int level, int finest_level, int nrk,
                       std::unique_ptr<amrex::MultiFab>& az,
                       std::unique_ptr<amrex::MultiFab>& dJ,
                       const amrex::MultiFab* p0,
-#ifdef ERF_USE_POISSON_SOLVE
                       const amrex::MultiFab& pp_inc,
-#endif
                       std::unique_ptr<amrex::MultiFab>& mapfac_m,
                       std::unique_ptr<amrex::MultiFab>& mapfac_u,
                       std::unique_ptr<amrex::MultiFab>& mapfac_v,

diff --git a/Source/TimeIntegration/ERF_TI_slow_rhs_fun.H b/Source/TimeIntegration/ERF_TI_slow_rhs_fun.H
@@ -140,9 +140,7 @@
                              Tau32_lev[level].get(), SmnSmn, eddyDiffs, Hfx1, Hfx2, Hfx3, Q1fx1, Q1fx2, Q1fx3, Q2fx3, Diss,
                              fine_geom, solverChoice, m_most, domain_bcs_type_d, domain_bcs_type,
                              z_phys_nd_src[level], ax_src[level], ay_src[level], az_src[level], detJ_cc_src[level], p0_new,
-#ifdef ERF_USE_POISSON_SOLVE
                              pp_inc[level],
-#endif
                              mapfac_m[level], mapfac_u[level], mapfac_v[level],
 #ifdef ERF_USE_EB
                              EBFactory(level),
@@ -251,9 +249,7 @@
                              Tau32_lev[level].get(), SmnSmn, eddyDiffs, Hfx1, Hfx2, Hfx3, Q1fx1, Q1fx2, Q1fx3,Q2fx3, Diss,
                              fine_geom, solverChoice, m_most, domain_bcs_type_d, domain_bcs_type,
                              z_phys_nd[level], ax[level], ay[level], az[level], detJ_cc[level], p0,
-#ifdef ERF_USE_POISSON_SOLVE
                              pp_inc[level],
-#endif
                              mapfac_m[level], mapfac_u[level], mapfac_v[level],
 #ifdef ERF_USE_EB
                              EBFactory(level),
@@ -459,9 +455,7 @@
                          Tau32_lev[level].get(), SmnSmn, eddyDiffs, Hfx1, Hfx2, Hfx3, Q1fx1, Q1fx2, Q1fx3, Q2fx3, Diss,
                          fine_geom, solverChoice, m_most, domain_bcs_type_d, domain_bcs_type,
                          z_phys_nd[level], ax[level], ay[level], az[level], detJ_cc[level], p0,
-#ifdef ERF_USE_POISSON_SOLVE
                          pp_inc[level],
-#endif
                          mapfac_m[level], mapfac_u[level], mapfac_v[level],
 #ifdef ERF_USE_EB
                          EBFactory(level),

diff --git a/Source/TimeIntegration/ERF_slow_rhs_pre.cpp b/Source/TimeIntegration/ERF_slow_rhs_pre.cpp
@@ -107,9 +107,7 @@ void erf_slow_rhs_pre (int level, int finest_level,
                        std::unique_ptr<MultiFab>& az,
                        std::unique_ptr<MultiFab>& detJ,
                        const MultiFab* p0,
-#ifdef ERF_USE_POISSON_SOLVE
                        const MultiFab& pp_inc,
-#endif
                        std::unique_ptr<MultiFab>& mapfac_m,
                        std::unique_ptr<MultiFab>& mapfac_u,
                        std::unique_ptr<MultiFab>& mapfac_v,
@@ -360,11 +358,7 @@ void erf_slow_rhs_pre (int level, int finest_level,
             });
         }
 
-#ifdef ERF_USE_POISSON_SOLVE
         const Array4<const Real>& pp_arr = (l_anelastic) ? pp_inc.const_array(mfi) : pprime.const_array();
-#else
-        const Array4<const Real>& pp_arr = pprime.const_array();
-#endif
 
         // *****************************************************************************
         // Contravariant flux field

diff --git a/Source/Utils/ERF_PoissonSolve.cpp b/Source/Utils/ERF_PoissonSolve.cpp
@@ -1,10 +1,8 @@
 #include "ERF.H"
 #include "ERF_Utils.H"
 
-#ifdef ERF_USE_POISSON_SOLVE
 #include <AMReX_MLMG.H>
 #include <AMReX_MLPoisson.H>
-#endif
 
 using namespace amrex;
 
@@ -49,7 +47,6 @@ bool ERF::projection_has_dirichlet (Array<LinOpBCType,AMREX_SPACEDIM> bcs) const
  */
 void ERF::project_velocities (int lev, Real l_dt, Vector<MultiFab>& mom_mf, MultiFab& pmf)
 {
-#ifdef ERF_USE_POISSON_SOLVE
     BL_PROFILE("ERF::project_velocities()");
 
     AMREX_ALWAYS_ASSERT(!solverChoice.use_terrain);
@@ -288,10 +285,4 @@ void ERF::project_velocities (int lev, Real l_dt, Vector<MultiFab>& mom_mf, Mult
         computeDivergence(rhs[0], rho0_u_const, geom_tmp[0]);
         Print() << "Max norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << std::endl;
     }
-#else
-    amrex::ignore_unused(lev);
-    amrex::ignore_unused(l_dt);
-    amrex::ignore_unused(mom_mf);
-    amrex::ignore_unused(pmf);
-#endif
 }