Merge branch 'development' into cleanup_smag_datastruct

erf-model · Jan 7, 2025 · 67ce9a4 · 67ce9a4
2 parents 8bc0a23 + 6f94c65
commit 67ce9a4
Show file tree

Hide file tree

Showing 36 changed files with 430 additions and 480 deletions.
diff --git a/Build/cmake_with_eb.sh b/Build/cmake_with_eb.sh
diff --git a/CMake/BuildERFExe.cmake b/CMake/BuildERFExe.cmake
@@ -35,19 +35,6 @@ function(build_erf_lib erf_lib_name)
     target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_PARTICLES)
   endif()
 
-  if(ERF_ENABLE_EB)
-    target_sources(${erf_lib_name} PRIVATE
-                   ${SRC_DIR}/EB/ERF_InitEB.cpp
-                   ${SRC_DIR}/EB/ERF_EBBox.cpp
-                   ${SRC_DIR}/EB/ERF_EBCylinder.cpp
-                   ${SRC_DIR}/EB/ERF_EBRegular.cpp
-                   ${SRC_DIR}/EB/ERF_InitEB.cpp
-                   ${SRC_DIR}/EB/ERF_WriteEBSurface.cpp 
-                   ${SRC_DIR}/LinearSolvers/ERF_SolveWithEBMLMG.cpp)
-    target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/EB>)
-    target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_EB)
-  endif()
-
   if(ERF_ENABLE_FFT)
     target_sources(${erf_lib_name} PRIVATE
                    ${SRC_DIR}/LinearSolvers/ERF_SolveWithFFT.cpp)
@@ -138,6 +125,11 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/Diffusion/ERF_ComputeStrain_N.cpp
        ${SRC_DIR}/Diffusion/ERF_ComputeStrain_T.cpp
        ${SRC_DIR}/Diffusion/ERF_ComputeTurbulentViscosity.cpp
+       ${SRC_DIR}/EB/ERF_InitEB.cpp
+       ${SRC_DIR}/EB/ERF_EBBox.cpp
+       ${SRC_DIR}/EB/ERF_EBRegular.cpp
+       ${SRC_DIR}/EB/ERF_InitEB.cpp
+       ${SRC_DIR}/EB/ERF_WriteEBSurface.cpp 
        ${SRC_DIR}/Initialization/ERF_InitBCs.cpp
        ${SRC_DIR}/Initialization/ERF_InitCustom.cpp
        ${SRC_DIR}/Initialization/ERF_InitFromHSE.cpp
@@ -192,6 +184,7 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/LinearSolvers/ERF_PoissonSolve_tb.cpp
        ${SRC_DIR}/LinearSolvers/ERF_PoissonWallDist.cpp
        ${SRC_DIR}/LinearSolvers/ERF_ComputeDivergence.cpp 
+       ${SRC_DIR}/LinearSolvers/ERF_SolveWithEBMLMG.cpp
        ${SRC_DIR}/LinearSolvers/ERF_SolveWithGMRES.cpp
        ${SRC_DIR}/LinearSolvers/ERF_SolveWithMLMG.cpp
        ${SRC_DIR}/LinearSolvers/ERF_TerrainPoisson.cpp
@@ -247,6 +240,7 @@ endif()
   target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/BoundaryConditions>)
   target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/DataStructs>)
   target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/Diffusion>)
+  target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/EB>)
   target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/Initialization>)
   target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/IO>)
   target_include_directories(${erf_lib_name} PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/LinearSolvers>)

diff --git a/CMake/SetAmrexOptions.cmake b/CMake/SetAmrexOptions.cmake
@@ -26,6 +26,7 @@ set(AMReX_CUDA ${ERF_ENABLE_CUDA})
 set(AMReX_ACC OFF)
 set(AMReX_PLOTFILE_TOOLS ${ERF_ENABLE_FCOMPARE})
 set(AMReX_FORTRAN OFF)
+set(AMReX_EB ON)
 
 set(AMReX_LINEAR_SOLVERS ON)
 set(AMReX_LINEAR_SOLVERS_EM OFF)
@@ -41,11 +42,6 @@ if(ERF_ENABLE_FFT)
   set(AMReX_FFT ON)
 endif()
 
-set(AMReX_EB OFF)
-if(ERF_ENABLE_EB)
-  set(AMReX_EB ON)
-endif()
-
 if(ERF_ENABLE_CUDA)
   set(AMReX_GPU_BACKEND CUDA CACHE STRING "AMReX GPU type" FORCE)
   set(AMReX_CUDA_WARN_CAPTURE_THIS OFF)

diff --git a/Exec/DevTests/EB_Test/ERF_Prob.cpp b/Exec/DevTests/EB_Test/ERF_Prob.cpp
@@ -169,14 +169,17 @@ Problem::init_custom_pert(
     dxInv[2] = 1. / dx[2];
 
     // Set the z-velocity from impenetrable condition
-    ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
-    {
-#ifdef ERF_USE_EB
-        z_vel_pert(i, j, k) = 0.0;
-#else
-        z_vel_pert(i, j, k) = WFromOmega(i, j, k, 0.0, x_vel_pert, y_vel_pert, z_nd, dxInv);
-#endif
-    });
+    if (sc.terrain_type == TerrainType::EB) {
+        ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
+        {
+            z_vel_pert(i, j, k) = 0.0;
+        });
+    } else {
+        ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
+        {
+            z_vel_pert(i, j, k) = WFromOmega(i, j, k, 0.0, x_vel_pert, y_vel_pert, z_nd, dxInv);
+        });
+    }
 
     amrex::Gpu::streamSynchronize();
 }

diff --git a/Exec/DevTests/EB_Test/GNUmakefile b/Exec/DevTests/EB_Test/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-USE_EB = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/Make.ERF b/Exec/Make.ERF
@@ -2,66 +2,59 @@ AMREX_HOME      ?= $(ERF_HOME)/Submodules/AMReX
 
 BL_NO_FORT = TRUE
 
+USE_EB = TRUE
+
 include $(AMREX_HOME)/Tools/GNUMake/Make.defs
 
 EBASE = ERF
 
 ERF_SOURCE_DIR = $(ERF_HOME)/Source
 include $(ERF_SOURCE_DIR)/Make.package
-
 VPATH_LOCATIONS   += $(ERF_SOURCE_DIR)
 INCLUDE_LOCATIONS += $(ERF_SOURCE_DIR)
 
 ERF_BC_DIR = $(ERF_SOURCE_DIR)/BoundaryConditions
 include $(ERF_BC_DIR)/Make.package
-
 VPATH_LOCATIONS   += $(ERF_BC_DIR)
 INCLUDE_LOCATIONS += $(ERF_BC_DIR)
 
 ERF_ADVECTION_DIR = $(ERF_SOURCE_DIR)/Advection
 include $(ERF_ADVECTION_DIR)/Make.package
+VPATH_LOCATIONS   += $(ERF_ADVECTION_DIR)
+INCLUDE_LOCATIONS += $(ERF_ADVECTION_DIR)
 
 ERF_DIFFUSION_DIR = $(ERF_SOURCE_DIR)/Diffusion
 include $(ERF_DIFFUSION_DIR)/Make.package
+VPATH_LOCATIONS   += $(ERF_DIFFUSION_DIR)
+INCLUDE_LOCATIONS += $(ERF_DIFFUSION_DIR)
 
 ERF_PBL_DIR = $(ERF_SOURCE_DIR)/PBL
 include $(ERF_PBL_DIR)/Make.package
+VPATH_LOCATIONS   += $(ERF_PBL_DIR)
+INCLUDE_LOCATIONS += $(ERF_PBL_DIR)
 
 ERF_INIT_DIR = $(ERF_SOURCE_DIR)/Initialization
 include $(ERF_INIT_DIR)/Make.package
+VPATH_LOCATIONS   += $(ERF_INIT_DIR)
+INCLUDE_LOCATIONS += $(ERF_INIT_DIR)
 
 ERF_DATA_DIR = $(ERF_SOURCE_DIR)/DataStructs
 include $(ERF_DATA_DIR)/Make.package
+VPATH_LOCATIONS   += $(ERF_DATA_DIR)
+INCLUDE_LOCATIONS += $(ERF_DATA_DIR)
 
 ERF_SOLVERS_DIR = $(ERF_SOURCE_DIR)/LinearSolvers
 include $(ERF_SOLVERS_DIR)/Make.package
+VPATH_LOCATIONS   += $(ERF_SOLVERS_DIR)
+INCLUDE_LOCATIONS += $(ERF_SOLVERS_DIR)
 
 ERF_UTIL_DIR = $(ERF_SOURCE_DIR)/Utils
 include $(ERF_UTIL_DIR)/Make.package
-
-ERF_MULTIBLOCK_DIR = $(ERF_SOURCE_DIR)/MultiBlock
-include $(ERF_MULTIBLOCK_DIR)/Make.package
-
-VPATH_LOCATIONS   += $(ERF_ADVECTION_DIR)
-INCLUDE_LOCATIONS += $(ERF_ADVECTION_DIR)
-
-VPATH_LOCATIONS   += $(ERF_DIFFUSION_DIR)
-INCLUDE_LOCATIONS += $(ERF_DIFFUSION_DIR)
-
-VPATH_LOCATIONS   += $(ERF_PBL_DIR)
-INCLUDE_LOCATIONS += $(ERF_PBL_DIR)
-
-VPATH_LOCATIONS   += $(ERF_INIT_DIR)
-INCLUDE_LOCATIONS += $(ERF_INIT_DIR)
-
-VPATH_LOCATIONS   += $(ERF_DATA_DIR)
-INCLUDE_LOCATIONS += $(ERF_DATA_DIR)
-
 VPATH_LOCATIONS   += $(ERF_UTIL_DIR)
 INCLUDE_LOCATIONS += $(ERF_UTIL_DIR)
 
-VPATH_LOCATIONS   += $(ERF_SOLVERS_DIR)
-INCLUDE_LOCATIONS += $(ERF_SOLVERS_DIR)
+ERF_MULTIBLOCK_DIR = $(ERF_SOURCE_DIR)/MultiBlock
+include $(ERF_MULTIBLOCK_DIR)/Make.package
 
 ifeq ($(USE_MULTIBLOCK),TRUE)
 VPATH_LOCATIONS   += $(ERF_MULTIBLOCK_DIR)
@@ -75,60 +68,30 @@ VPATH_LOCATIONS   += $(ERF_PARTICLES_DIR)
 INCLUDE_LOCATIONS += $(ERF_PARTICLES_DIR)
 endif
 
-ifeq ($(USE_EB),TRUE)
+include $(ERF_PROBLEM_DIR)/Make.package
+VPATH_LOCATIONS   += $(ERF_PROBLEM_DIR)
+INCLUDE_LOCATIONS += $(ERF_PROBLEM_DIR)
+
 ERF_EB_DIR = $(ERF_SOURCE_DIR)/EB
 include $(ERF_EB_DIR)/Make.package
 VPATH_LOCATIONS   += $(ERF_EB_DIR)
 INCLUDE_LOCATIONS += $(ERF_EB_DIR)
-endif
 
 ERF_SRCTERMS_DIR = $(ERF_SOURCE_DIR)/SourceTerms
 include $(ERF_SRCTERMS_DIR)/Make.package
-
 VPATH_LOCATIONS   += $(ERF_SRCTERMS_DIR)
 INCLUDE_LOCATIONS += $(ERF_SRCTERMS_DIR)
 
 ERF_TIMEINT_DIR = $(ERF_SOURCE_DIR)/TimeIntegration
 include $(ERF_TIMEINT_DIR)/Make.package
-
 VPATH_LOCATIONS   += $(ERF_TIMEINT_DIR)
 INCLUDE_LOCATIONS += $(ERF_TIMEINT_DIR)
 
 ERF_IO_DIR = $(ERF_SOURCE_DIR)/IO
 include $(ERF_IO_DIR)/Make.package
-
 VPATH_LOCATIONS   += $(ERF_IO_DIR)
 INCLUDE_LOCATIONS += $(ERF_IO_DIR)
 
-include $(ERF_PROBLEM_DIR)/Make.package
-
-VPATH_LOCATIONS   += $(ERF_PROBLEM_DIR)
-INCLUDE_LOCATIONS += $(ERF_PROBLEM_DIR)
-
-include $(AMREX_HOME)/Src/Base/Make.package
-
-AMReXdirs             := Base Boundary AmrCore
-
-ifeq ($(USE_PARTICLES),TRUE)
-AMReXdirs             += Particle
-endif
-
-ifeq ($(USE_EB),TRUE)
-AMReXdirs             += EB
-endif
-
-USE_LINEAR_SOLVERS_INCFLO = TRUE
-USE_LINEAR_SOLVERS_EM     = FALSE
-AMReXdirs                += LinearSolvers
-
-ifeq ($(USE_FFT),TRUE)
-AMReXdirs                += FFT
-endif
-
-AMReXpack             += $(foreach dir, $(AMReXdirs), $(AMREX_HOME)/Src/$(dir)/Make.package)
-
-include $(AMReXpack)
-
 ERF_MOISTURE_DIR = $(ERF_SOURCE_DIR)/Microphysics
 include $(ERF_MOISTURE_DIR)/Make.package
 VPATH_LOCATIONS   += $(ERF_MOISTURE_DIR)
@@ -238,6 +201,30 @@ ERF_LSM_MM5_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/MM5
 include $(ERF_LSM_MM5_DIR)/Make.package
 VPATH_LOCATIONS   += $(ERF_LSM_MM5_DIR)
 INCLUDE_LOCATIONS += $(ERF_LSM_MM5_DIR)
+
+include $(AMREX_HOME)/Src/Base/Make.package
+
+AMReXdirs             := Base Boundary AmrCore
+
+ifeq ($(USE_PARTICLES),TRUE)
+AMReXdirs             += Particle
+endif
+
+ifeq ($(USE_EB),TRUE)
+AMReXdirs             += EB
+endif
+
+USE_LINEAR_SOLVERS_INCFLO = TRUE
+USE_LINEAR_SOLVERS_EM     = FALSE
+AMReXdirs                += LinearSolvers
+
+ifeq ($(USE_FFT),TRUE)
+AMReXdirs                += FFT
+endif
+
+AMReXpack             += $(foreach dir, $(AMReXdirs), $(AMREX_HOME)/Src/$(dir)/Make.package)
+
+include $(AMReXpack)
 
 ifeq ($(COMPUTE_ERROR), TRUE)
   DEFINES += -DERF_COMPUTE_ERROR
@@ -247,10 +234,6 @@ ifeq ($(USE_PARTICLES), TRUE)
   DEFINES += -DERF_USE_PARTICLES
 endif
 
-ifeq ($(USE_EB), TRUE)
-  DEFINES += -DERF_USE_EB
-endif
-
 ifeq ($(USE_MULTIBLOCK), TRUE)
   DEFINES += -DERF_USE_MULTIBLOCK
 endif

diff --git a/Source/Advection/ERF_Advection.H b/Source/Advection/ERF_Advection.H
@@ -5,6 +5,7 @@
 #include <AMReX_MultiFab.H>
 #include <AMReX_BCRec.H>
 #include <AMReX_YAFluxRegister.H>
+
 #include <ERF_DataStruct.H>
 #include <ERF_IndexDefines.H>
 #include <ERF_ABLMost.H>
@@ -74,7 +75,8 @@ void AdvectionSrcForMom (const amrex::Box& bx,
                          const amrex::Array4<const amrex::Real>& mf_v,
                          const AdvType horiz_adv_type, const AdvType vert_adv_type,
                          const amrex::Real horiz_upw_frac, const amrex::Real vert_upw_frac,
-                         const bool use_terrain, const int lo_z_face, const int hi_z_face,
+                         TerrainType& terrain_type,
+                         const int lo_z_face, const int hi_z_face,
                          const amrex::Box& domain,
                          const amrex::BCRec* bc_ptr_h);