diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 47ebacc4e8f4..3797070573c7 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -26,7 +26,7 @@ on: jobs: ci: - if: ${{ github.event.repository.name == 'e3sm' }} + if: false runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index 950f335700ee..d39717589914 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -13,6 +13,7 @@ on: jobs: scream-defaults: + if: false runs-on: ubuntu-latest outputs: event_name: ${{ github.event_name }} diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index b92fe082d7ea..7be965a19491 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -1783,6 +1783,26 @@ oRRS18to6v3 + + ne512np4.pg2 + ne512np4.pg2 + ICOS10 + r0125 + null + null + ICOS10 + + + + ne512np4.pg2 + ne512np4.pg2 + ne512np4.pg2 + r0125 + null + null + ICOS10 + + ne512np4 360x720cru @@ -3082,6 +3102,8 @@ 1 $DIN_LOC_ROOT/share/domains/domain.lnd.ne512pg2_oRRS18to6v3.200212.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne512pg2_oRRS18to6v3.200212.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne512pg2_ICOS10.240602.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne512pg2_ICOS10.240602.nc ne512np4.pg2 is Spectral Elem 6km grid w/ 2x2 FV physics grid per element: @@ -4187,6 +4209,11 @@ cpl/gridmaps/ne256pg2/map_ne256pg2_to_r0125_bilin.200212.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_oRRS18to6v3_nco.200212.nc @@ -4196,6 +4223,15 @@ cpl/gridmaps/ne512pg2/map_oRRS18to6v3_to_ne512pg2_nco.200212.nc + + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ICOS10_to_ne512pg2_nco_c240531.nc + cpl/gridmaps/ne512pg2/map_ICOS10_to_ne512pg2_nco_c240531.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.200212.nc @@ -5002,6 +5038,11 @@ cpl/gridmaps/ne256pg2/map_r0125_to_ne256pg2_mono.200212.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + cpl/gridmaps/ne512pg2/map_r0125_to_ne512pg2_mono.c20240625.nc + + cpl/gridmaps/ne1024pg2/map_ne1024pg2_to_r0125_mono.200212.nc cpl/gridmaps/ne1024pg2/map_r0125_to_ne1024pg2_mono.200212.nc @@ -5082,6 +5123,11 @@ lnd/clm2/mappingdata/maps/ne240np4/map_0.5x0.5_nomask_to_ne240np4_nomask_aave_da_c121019.nc + + cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc + cpl/gridmaps/ne512pg2/map_r0125_to_ne512pg2_mono.c20240625.nc + + cpl/gridmaps/ne1024pg2/map_ne1024pg2_to_r0125_mono.200212.nc cpl/gridmaps/ne1024pg2/map_r0125_to_ne1024pg2_mono.200212.nc diff --git a/cime_config/machines/Depends.crusher-gpu.crayclang.cmake b/cime_config/machines/Depends.crusher-gpu.crayclang.cmake deleted file mode 100644 index e41d959b52b4..000000000000 --- a/cime_config/machines/Depends.crusher-gpu.crayclang.cmake +++ /dev/null @@ -1,43 +0,0 @@ -set(CICE_F90 - ice_FY.F90 - ice_aerosol.F90 - ice_age.F90 - ice_atmo.F90 - ice_blocks.F90 - ice_calendar.F90 - ice_diagnostics.F90 - ice_distribution.F90 - ice_domain.F90 - ice_domain_size.F90 - ice_dyn_evp.F90 - ice_fileunits.F90 - ice_flux.F90 - ice_forcing.F90 - ice_grid.F90 - ice_history.F90 - ice_history_fields.F90 - ice_init.F90 - ice_itd.F90 - ice_kinds_mod.F90 - ice_lvl.F90 - ice_mechred.F90 - ice_meltpond.F90 - ice_ocean.F90 - ice_orbital.F90 - ice_probability.F90 - ice_probability_tools.F90 - ice_read_write.F90 - ice_restoring.F90 - ice_shortwave.F90 - ice_spacecurve.F90 - ice_state.F90 - ice_step_mod.F90 - ice_therm_itd.F90 - ice_therm_vertical.F90 - ice_transport_driver.F90 - ice_transport_remap.F90 - ice_work.F90) - -foreach(ITEM IN LISTS CICE_F90) - e3sm_add_flags("cice/src/source/${ITEM}" "-O0") -endforeach() diff --git a/cime_config/machines/cmake_macros/amdclang_crusher.cmake b/cime_config/machines/cmake_macros/amdclang_crusher.cmake deleted file mode 100644 index 2df5074d11f3..000000000000 --- a/cime_config/machines/cmake_macros/amdclang_crusher.cmake +++ /dev/null @@ -1,12 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") diff --git a/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake b/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake deleted file mode 100644 index 6f3c0074798e..000000000000 --- a/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake +++ /dev/null @@ -1,26 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") - -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") - -set(MPICXX "hipcc") -set(SCXX "hipcc") -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake deleted file mode 100644 index f8176ccb3d0b..000000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake +++ /dev/null @@ -1,12 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() - -set(PIO_FILESYSTEM_HINTS "gpfs") - -string(APPEND CPPDEFS " -DCPRCRAY") - - diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake deleted file mode 100644 index 18564d4d301f..000000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake +++ /dev/null @@ -1,28 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() - -set(PIO_FILESYSTEM_HINTS "gpfs") - -set(MPICXX "hipcc") -set(SCXX "hipcc") - -string(APPEND CMAKE_CXX_FLAGS " -I${MPICH_DIR}/include") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L${MPICH_DIR}/lib -lmpi -L/opt/cray/pe/mpich/8.1.16/gtl/lib -lmpi_gtl_hsa") - -# For YAKL's -lroctx64 -lrocfft; the rocm module doesn't set this. -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib") - -# 'NOT DEBUG': this resolves a crash in mct in docn init -# 'DEBUG' casee, too: resolves a build error in elm/src/main/elm_varctl.F90 due to several OpenACC syntax errors -#if (NOT DEBUG) - string(APPEND CMAKE_C_FLAGS " -O2 -hnoacc -hfp0 -hipa0") - string(APPEND CMAKE_Fortran_FLAGS " -O2 -hnoacc -hfp0 -hipa0") -#endif() - -string(APPEND CPPDEFS " -DCPRCRAY") - -#set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "See SCREAM issue #2080.") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake deleted file mode 100644 index 2a0bfd6217c8..000000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake +++ /dev/null @@ -1,12 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() -if (COMP_NAME STREQUAL elm) - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") - -set(PIO_FILESYSTEM_HINTS "gpfs") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake index afcca8f479e5..d42597be2e0b 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -5,7 +5,7 @@ set(SCC "cc") set(SCXX "hipcc") set(SFC "ftn") -string(APPEND CPPDEFS " -DLINUX") +string(APPEND CPPDEFS " -DLINUX -DSCREAM_SYSTEM_WORKAROUND=0") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() @@ -19,7 +19,7 @@ endif() string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -f free") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64 -L/opt/gcc/12.2.0/snos/lib64") string(APPEND CMAKE_CXX_FLAGS " -I$ENV{ROCM_PATH}/include") # Crusher: this resolves a crash in mct in docn init diff --git a/cime_config/machines/cmake_macros/crayclang_crusher.cmake b/cime_config/machines/cmake_macros/crayclang_crusher.cmake deleted file mode 100644 index 7a5fb412cbb9..000000000000 --- a/cime_config/machines/cmake_macros/crayclang_crusher.cmake +++ /dev/null @@ -1,10 +0,0 @@ -if (COMP_NAME STREQUAL elm) - # See Land NaNs in conditionals: https://github.com/E3SM-Project/E3SM/issues/4996 - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -# Disable ipa and zero initialization are for other NaN isues: -# https://github.com/E3SM-Project/E3SM/pull/5208 -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") -# -em -ef generates modulename.mod (lowercase files) to support -# Scorpio installs -string(APPEND CMAKE_Fortran_FLAGS " -em -ef") diff --git a/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake b/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake deleted file mode 100644 index 653938b81da9..000000000000 --- a/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake +++ /dev/null @@ -1,63 +0,0 @@ -set(MPICC "cc") -set(MPICXX "hipcc") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "hipcc") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() - -if (compile_threaded) - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() -string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g") -string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") -string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG") -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY") -string(APPEND CMAKE_Fortran_FLAGS " -f free -em") -if (NOT compile_threaded) - # -M1077 flag used to suppress message about OpenMP directives - # that are ignored for non-threaded builds. (-h omp inactive) - # Details: `explain ftn-1077` - string(APPEND CMAKE_Fortran_FLAGS " -M1077") -endif() -set(HAS_F2008_CONTIGUOUS "TRUE") - -# -Wl,--allow-shlib-undefined was added to address rocm 5.4.3 Fortran linker issue: -# /opt/rocm-5.4.3/lib/libhsa-runtime64.so.1: undefined reference to `std::condition_variable::wait(std::unique_lock&)@GLIBCXX_3.4.30' -# AMD started building with GCC 12.2.0, which brings in a GLIBCXX symbol that isn't in CCE's default GCC toolchain. -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined") - -# Switch to O3 for better performance -# Using O2 to ensure passing tests -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") - -if (COMP_NAME STREQUAL elm) - # See Land NaNs in conditionals: https://github.com/E3SM-Project/E3SM/issues/4996 - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -# -em -ef generates modulename.mod (lowercase files) to support -# Scorpio installs -# Disable ipa and zero initialization are for other NaN isues: -# https://github.com/E3SM-Project/E3SM/pull/5208 -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc") - -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") - -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/cmake_macros/craygnuamdgpu.cmake b/cime_config/machines/cmake_macros/craygnuamdgpu.cmake new file mode 100644 index 000000000000..6cb79c0146b1 --- /dev/null +++ b/cime_config/machines/cmake_macros/craygnuamdgpu.cmake @@ -0,0 +1,42 @@ +set(MPICC "cc") +set(MPICXX "hipcc") # Needs MPICH_CXX to use hipcc +set(MPIFC "ftn") # Linker needs to be the Cray wrapper ftn, not mpif90 +set(SCC "cc") +set(SCXX "hipcc") +set(SFC "ftn") + +string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU -DSCREAM_SYSTEM_WORKAROUND_P3_PART2") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND CMAKE_Fortran_FLAGS " -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none -fallow-argument-mismatch") + +string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=invalid,zero,overflow") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=zero,overflow") +string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace") + +string(APPEND CMAKE_C_FLAGS_RELEASE " -g -O2") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -g -O2") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -g -O2") + +if (COMP_NAME STREQUAL csm_share) + string(APPEND CMAKE_C_FLAGS " -std=c99") +endif() +string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -ffixed-form") +string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -ffree-form") + +set(E3SM_LINK_WITH_FORTRAN "TRUE") +string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") + +if (compile_threaded) + string(APPEND CMAKE_C_FLAGS " -fopenmp") + string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") + string(APPEND CMAKE_CXX_FLAGS " -fopenmp=libgomp") + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") +endif() + +string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off") + +set(USE_HIP "TRUE") +string(APPEND CMAKE_HIP_FLAGS "$ENV{CXXFLAGS} --offload-arch=gfx90a -munsafe-fp-atomics") diff --git a/cime_config/machines/cmake_macros/gnu_crusher.cmake b/cime_config/machines/cmake_macros/gnu_crusher.cmake deleted file mode 100644 index 9b242243989b..000000000000 --- a/cime_config/machines/cmake_macros/gnu_crusher.cmake +++ /dev/null @@ -1,15 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") - -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") diff --git a/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake b/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake index 226d07350a78..c2093f7f7fbb 100644 --- a/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake +++ b/cime_config/machines/cmake_macros/gnu_pm-cpu.cmake @@ -10,3 +10,6 @@ set(MPIFC "ftn") set(SCC "gcc") set(SCXX "g++") set(SFC "gfortran") + +string(APPEND CMAKE_EXE_LINKER_FLAGS " -static-libstdc++") + diff --git a/cime_config/machines/cmake_macros/gnugpu_crusher.cmake b/cime_config/machines/cmake_macros/gnugpu_crusher.cmake deleted file mode 100644 index bb213014b2f5..000000000000 --- a/cime_config/machines/cmake_macros/gnugpu_crusher.cmake +++ /dev/null @@ -1,25 +0,0 @@ -set(MPICC "cc") -set(MPICXX "hipcc") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "hipcc") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") - -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--copy-dt-needed-entries -L/opt/cray/pe/gcc-libs -lgfortran -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ") - -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 140b733f83db..7af65ddcccbf 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -382,6 +382,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -392,24 +393,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/22.7 + nvidia/24.5 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 - gcc-mixed/11.2.0 + gcc-native-mixed/12.3 @@ -421,12 +422,12 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 + cray-libsci/23.12.5 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 @@ -454,6 +455,9 @@ 1 + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnugpu ; else echo "$MOAB_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} @@ -685,6 +689,7 @@ nvidia aocc cudatoolkit + cray-libsci climate-utils matlab craype-accel-nvidia80 @@ -695,26 +700,24 @@ - PrgEnv-gnu/8.3.3 - gcc/11.2.0 - + PrgEnv-gnu/8.5.0 + gcc-native/12.3 PrgEnv-nvidia - nvidia/23.9 + nvidia/24.5 - cudatoolkit/11.7 - + cudatoolkit/12.2 craype-accel-nvidia80 - cudatoolkit/11.7 + cudatoolkit/12.2 craype-accel-nvidia80 + gcc-native-mixed/12.3 @@ -726,20 +729,13 @@ - cray-libsci/23.02.1.1 - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 - evp-patch @@ -1115,306 +1111,11 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss --> - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu - mpich - cli115 - /lustre/orion/cli115/world-shared/crusher - .* - /lustre/orion/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli115/world-shared/e3sm/baselines/crusher/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 56 - 56 - 8 - 8 - 8 - TRUE - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - -c $ENV{OMP_NUM_THREADS} - $ENV{NTASKS_PER_GPU} - $ENV{GPU_BIND_ARGS} - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - PrgEnv-cray PrgEnv-cray/8.3.3 - cce cce/15.0.1 - - - craype craype/2.7.20 - - - craype-accel-amd-gfx90a - rocm/5.4.0 - - - - PrgEnv-cray PrgEnv-amd/8.3.3 - amd amd/5.4.0 - - - craype-accel-amd-gfx90a - - - - PrgEnv-cray PrgEnv-gnu/8.3.3 - gcc gcc/11.2.0 - - - craype-accel-amd-gfx90a - rocm/5.4.0 - - - cray-python/3.9.13.1 - subversion/1.14.1 - git/2.36.1 - cmake/3.21.3 - zlib/1.2.11 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - 0.25 - 0 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - - - - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - - - --ntasks-per-gpu=$SHELL{echo "`./xmlquery --value MAX_MPITASKS_PER_NODE`/8"|bc} - --gpu-bind=closest - romio_cb_read=disable - 0 - - - 10 - 3 - - - 128M - spread - threads - - - - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang-scream - mpich - CLI115 - /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 56 - 56 - TRUE - - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - - - --threads-per-core=1 - -c $ENV{OMP_NUM_THREADS} - -m *:block - - - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - - PrgEnv-cray PrgEnv-cray/8.3.3 - cce cce/14.0.0 - - - - - PrgEnv-cray PrgEnv-amd/8.3.3 - amd amd/5.1.0 - - - - - PrgEnv-cray PrgEnv-gnu/8.3.3 - - - cray-mpich/8.1.12 - cray-python/3.9.4.2 - subversion/1.14.0 - git/2.31.1 - cmake/3.21.3 - zlib/1.2.11 - cray-libsci/21.08.1.2 - cray-hdf5-parallel/1.12.1.1 - cray-netcdf-hdf5parallel/4.8.1.1 - cray-parallel-netcdf/1.12.1.7 - - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - - - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - - - - 0 - - - - 128M - spread - threads - - - - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang-scream - mpich - CLI115 - /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 64 - 8 - TRUE - - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - - - --gpus-per-node=8 --gpu-bind=closest - -c $ENV{OMP_NUM_THREADS} - - - - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - PrgEnv-cray - - craype-accel-amd-gfx90a - rocm/5.1.0 - - cce/14.0.3 - - - cray-python/3.9.4.2 - subversion/1.14.0 - git/2.31.1 - cmake/3.21.3 - zlib/1.2.11 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 - - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - 0 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - 0 - - 1 - romio_cb_read=disable - - - - 128M - spread - threads - - - Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. .*frontier.* CNL - crayclang-scream + craygnuamdgpu,crayclang-scream mpich cli115 /lustre/orion/proj-shared/cli115 @@ -1451,17 +1152,34 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss module module /usr/share/lmod/lmod/libexec/lmod python + + + + PrgEnv-gnu + cpe/24.07 + libfabric/1.15.2.0 + craype-accel-amd-gfx90a + rocm/6.2.0 + libunwind + cray-python + subversion + git + cmake + cray-hdf5-parallel + cray-netcdf-hdf5parallel + cray-parallel-netcdf + darshan-runtime + - PrgEnv-cray + cpe/22.12 craype-accel-amd-gfx90a rocm/5.4.0 libunwind/1.6.2 - - cce/15.0.1 - craype craype/2.7.20 - cray-mpich cray-mpich/8.1.26 + libfabric/1.15.2.0 + craype/2.7.20 + cray-mpich/8.1.26 cray-python/3.9.13.1 subversion/1.14.1 git/2.36.1 @@ -1495,9 +1213,13 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss threads + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.10/install/craygnuamdgppu/cpe-24.07/libfabric-1.15.2.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} + diff --git a/cime_config/machines/syslog.crusher b/cime_config/machines/syslog.crusher deleted file mode 100755 index 25ef50e1e955..000000000000 --- a/cime_config/machines/syslog.crusher +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/csh -f -# pm-gpu syslog script: -# mach_syslog - -set sample_interval = $1 -set jid = $2 -set lid = $3 -set run = $4 -set timing = $5 -set dir = $6 - -# Wait until job task-to-node mapping information is output before saving output file. -# Target length was determined empirically (maximum number of lines before job mapping -# information starts + number of nodes), and it may need to be adjusted in the future. -# (Note that calling script 'touch'es the e3sm log file before spawning this script, so that 'wc' does not fail.) -set nnodes = `scontrol show jobid $jid | grep -F NumNodes | sed 's/ *NumNodes=\([0-9]*\) .*/\1/' ` -@ target_lines = 150 + $nnodes -sleep 10 -set outlth = `wc \-l $run/e3sm.log.$lid | sed 's/ *\([0-9]*\) *.*/\1/' ` -while ($outlth < $target_lines) - sleep 60 - set outlth = `wc \-l $run/e3sm.log.$lid | sed 's/ *\([0-9]*\) *.*/\1/' ` -end - -set TimeLimit = `scontrol show jobid $jid | grep -F TimeLimit | sed 's/^ *RunTime=.*TimeLimit=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` -set limit_hours = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` -set limit_mins = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` -set limit_secs = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` -if ("X$limit_hours" == "X") set limit_hours = 0 -if ("X$limit_mins" == "X") set limit_mins = 0 -if ("X$limit_secs" == "X") set limit_secs = 0 -@ limit = 3600 * $limit_hours + 60 * $limit_mins + $limit_secs - -set RunTime = `scontrol show jobid $jid | grep -F RunTime | sed 's/^ *RunTime=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` -set runt_hours = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` -set runt_mins = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` -set runt_secs = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` -if ("X$runt_hours" == "X") set runt_hours = 0 -if ("X$runt_mins" == "X") set runt_mins = 0 -if ("X$runt_secs" == "X") set runt_secs = 0 -@ runt = 3600 * $runt_hours + 60 * $runt_mins + $runt_secs - -@ remaining = $limit - $runt -cat > $run/Walltime.Remaining < $dir/squeuef.$lid.$remaining - squeue -s | grep -v -F extern > $dir/squeues.$lid.$remaining - # squeue -t R -o "%.10i %R" > $dir/squeueR.$lid.$remaining -endif - -while ($remaining > 0) - echo "Wallclock time remaining: $remaining" >> $dir/atm.log.$lid.step - grep -Fa -e "nstep" -e "model date" $run/*atm.log.$lid | tail -n 4 >> $dir/atm.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/lnd.log.$lid.step - grep -Fa -e "timestep" -e "model date" $run/*lnd.log.$lid | tail -n 4 >> $dir/lnd.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/ocn.log.$lid.step - grep -Fa -e "timestep" -e "Step number" -e "model date" $run/*ocn.log.$lid | tail -n 4 >> $dir/ocn.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/ice.log.$lid.step - grep -Fa -e "timestep" -e "istep" -e "model date" $run/*ice.log.$lid | tail -n 4 >> $dir/ice.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/rof.log.$lid.step - grep -Fa "model date" $run/*rof.log.$lid | tail -n 4 >> $dir/rof.log.$lid.step - grep -Fa "model date" $run/*cpl.log.$lid > $dir/cpl.log.$lid.step-all - echo "Wallclock time remaining: $remaining" >> $dir/cpl.log.$lid.step - tail -n 4 $dir/cpl.log.$lid.step-all >> $dir/cpl.log.$lid.step - /bin/cp --preserve=timestamps -u $timing/* $dir - # sqs -w -a | grep "^[0-9]* *R *"> $dir/sqswr.$lid.$remaining - squeue -t R -o "%.10i %.15P %.20j %.10u %.7a %.2t %.6D %.8C %.10M %.10l" > $dir/squeuef.$lid.$remaining - squeue -s | grep -v -F extern > $dir/squeues.$lid.$remaining - # squeue -t R -o "%.10i %R" > $dir/squeueR.$lid.$remaining - chmod a+r $dir/* - # sleep $sample_interval - set sleep_remaining = $sample_interval - while ($sleep_remaining > 120) - sleep 120 - @ sleep_remaining = $sleep_remaining - 120 - end - sleep $sleep_remaining - set RunTime = `scontrol show jobid $jid | grep -F RunTime | sed 's/^ *RunTime=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` - set runt_hours = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` - set runt_mins = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` - set runt_secs = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` - if ("X$runt_hours" == "X") set runt_hours = 0 - if ("X$runt_mins" == "X") set runt_mins = 0 - if ("X$runt_secs" == "X") set runt_secs = 0 - @ runt = 3600 * $runt_hours + 60 * $runt_mins + $runt_secs - @ remaining = $limit - $runt - cat > $run/Walltime.Remaining << EOF2 -$remaining $sample_interval -EOF2 - -end diff --git a/cime_config/tests.py b/cime_config/tests.py index df2d97716767..cc73de81110b 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -678,13 +678,20 @@ "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero.scream-output-preset-3", "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-4", "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2--scream-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels_p3--scream-output-preset-5", + "ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels_shoc--scream-output-preset-5", ) }, "e3sm_scream_v1_dp-eamxx" : { "time" : "01:00:00", + # each test runs with 225 dynamics and 100 physics columns, roughly size of ne2 "tests" : ( - "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-DYCOMSrf01", # 225 phys cols, roughly size of ne2 + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-dycomsrf01", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-comble", + "ERS_P16_Ln22.ne30pg2_ne30pg2.FRCE-SCREAMv1-DP", ) }, @@ -752,6 +759,8 @@ "tests" : ( "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics", "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-wetscav", + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-drydep", ) }, diff --git a/components/eam/src/dynamics/se/dyn_comp.F90 b/components/eam/src/dynamics/se/dyn_comp.F90 index b420be4ee690..faff6ffd314b 100644 --- a/components/eam/src/dynamics/se/dyn_comp.F90 +++ b/components/eam/src/dynamics/se/dyn_comp.F90 @@ -269,7 +269,7 @@ subroutine dyn_init1(fh, NLFileName, dyn_in, dyn_out) end if #ifdef HAVE_MOAB - call create_moab_meshes(par, elem) + call create_moab_meshes(par, elem, fv_nphys) #endif ! Define the CAM grids (this has to be after dycore spinup). ! Physics-grid will be defined later by phys_grid_init diff --git a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 index edea27959d25..11d13c71a7a7 100644 --- a/components/eam/src/physics/p3/scream/micro_p3_interface.F90 +++ b/components/eam/src/physics/p3/scream/micro_p3_interface.F90 @@ -834,6 +834,8 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) real(rtype) :: icwmrst(pcols,pver) ! stratus water mixing ratio - on grid real(rtype) :: rho(pcols,pver) real(rtype) :: reff_rain(pcols,pver) + real(rtype) :: deff_rain(pcols,pver) + real(rtype) :: col_location(pcols,3),tmp_loc(pcols) ! Array of column lon (index 1) and lat (index 2) integer :: tmpi_loc(pcols) ! Global column index temp array @@ -1358,6 +1360,7 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) aqrain = 0._rtype anrain = 0._rtype freqr = 0._rtype + deff_rain = reff_rain*2._rtype*1.e6_rtype !rain effective diameter in microns ! Prognostic precipitation where (rain(:ncol,top_lev:) >= 1.e-7_rtype) aqrain(:ncol,top_lev:) = rain(:ncol,top_lev:) * cld_frac_r(:ncol,top_lev:) @@ -1412,6 +1415,7 @@ subroutine micro_p3_tend(state, ptend, dtime, pbuf) call outfld('FREQI', freqi, pcols, lchnk) call outfld('FREQR', freqr, pcols, lchnk) call outfld('CDNUMC', cdnumc, pcols, lchnk) + call outfld('ADRAIN', deff_rain, pcols, lchnk) call outfld('CLOUDFRAC_LIQ_MICRO', cld_frac_l, pcols, lchnk) call outfld('CLOUDFRAC_ICE_MICRO', cld_frac_i, pcols, lchnk) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index 4641dc245ba0..e64b99cce24e 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit 4641dc245ba00cb230c740f4ab3fd0d5a7543a6b +Subproject commit e64b99cce24eb31bb6f317bddb6f0ffbdfaf8bb7 diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 580ce26e7f50..07c3cda76728 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -198,8 +198,10 @@ set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C inst set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine") option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) -set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels") -if (NOT SCREAM_SMALL_KERNELS) +set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for ALL components that support them") +set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only") +set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only") +if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_SHOC_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") endif() @@ -272,8 +274,8 @@ set(SCREAM_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(SCREAM_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) set(SCREAM_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}) -option (EAMXX_ENABLE_PYBIND "Whether to enable python interface to eamxx, via pybind11" OFF) -if (EAMXX_ENABLE_PYBIND) +option (EAMXX_ENABLE_PYSCREAM "Whether to enable python interface to eamxx" OFF) +if (EAMXX_ENABLE_PYSCREAM) # Pybind11 requires shared libraries set (BUILD_SHARED_LIBS ON) endif() @@ -550,8 +552,7 @@ DisableMpiCxxBindings() if (SCREAM_DOUBLE_PRECISION) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") set(SCREAM_Fortran_FLAGS -real-size 64) - elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") - #find what is crayclang id in cmake + elseif("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Cray") set(SCREAM_Fortran_FLAGS -s default32 -eZ) else() set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8) diff --git a/components/eamxx/cime_config/config_component.xml b/components/eamxx/cime_config/config_component.xml index 0aeccd1e6d24..7380b371401d 100644 --- a/components/eamxx/cime_config/config_component.xml +++ b/components/eamxx/cime_config/config_component.xml @@ -60,6 +60,18 @@ + + + char + + + + $SRCROOT/components/eamxx/cime_config/usermods_dirs/rcemip + + run_component_cam + env_case.xml + User mods to apply to specific compset matches. + scream default diff --git a/components/eamxx/cime_config/config_compsets.xml b/components/eamxx/cime_config/config_compsets.xml index 6b7035e9c813..b378efcfdebe 100644 --- a/components/eamxx/cime_config/config_compsets.xml +++ b/components/eamxx/cime_config/config_compsets.xml @@ -75,8 +75,14 @@ - F2010-SCREAMv1-DP-DYCOMSrf01 - 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%DYCOMSrf01 + FIOP-SCREAMv1-DP + 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%IOP + Experimental, under development + + + + FRCE-SCREAMv1-DP + 2000_SCREAM%RCE_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx Experimental, under development @@ -86,62 +92,63 @@ 2016-08-01 2020-01-20 - 1999-07-10 + 2000-01-01 - + - 864 + 864 - TRUE + TRUE - TRUE + TRUE - + - 31.5 + 100 - + - 238.5 + 1 - + - 225 + 100 - + - 1 + 1 - + + - 225 + 0.0 - + - 1 + 0.0 diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index 0c7ed8e52aac..bf425c2279c7 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -4,7 +4,7 @@ Used by buildnml. See buildnml for documetation. """ -import os, sys, re +import os, sys, re, pwd, grp, stat, getpass from collections import OrderedDict import xml.etree.ElementTree as ET @@ -126,7 +126,7 @@ def perform_consistency_checks(case, xml): >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':2, 'REST_OPTION':'nsteps'}) >>> perform_consistency_checks(case,xml) Traceback (most recent call last): - CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency incompatible with restart frequency. + CIME.utils.CIMEError: ERROR: rrtmgp::rad_frequency (3 steps) incompatible with restart frequency (2 steps). Please, ensure restart happens on a step when rad is ON >>> case = MockCase({'ATM_NCPL':'24', 'REST_N':10800, 'REST_OPTION':'nseconds'}) >>> perform_consistency_checks(case,xml) @@ -181,7 +181,8 @@ def perform_consistency_checks(case, xml): pass elif rest_opt in ["nsteps", "nstep"]: expect (rest_n % rad_freq == 0, - "rrtmgp::rad_frequency incompatible with restart frequency.\n" + f"rrtmgp::rad_frequency ({rad_freq} steps) incompatible with " + f"restart frequency ({rest_n} steps).\n" " Please, ensure restart happens on a step when rad is ON") elif rest_opt in ["nseconds", "nsecond", "nminutes", "nminute", "nhours", "nhour"]: if rest_opt in ["nseconds", "nsecond"]: @@ -952,7 +953,37 @@ def create_input_data_list_file(case,caseroot): # Only add files whose full path starts with the CIME's input data location if file_path.startswith(din_loc_root): fd.write("scream_dl_input_{} = {}\n".format(idx, file_path)) - + if os.path.exists(file_path): + if os.path.isdir(file_path): + raise IsADirectoryError(f"Input file '{file_path}' is a directory, not a regular file.") + if not os.path.isfile(file_path): + raise OSError(f"Input file '{file_path}' exists but is not a regular file.") + if not os.access(file_path,os.R_OK): + try: + file_stat = os.stat(file_path) + + # Get owner and group names + owner = pwd.getpwuid(file_stat.st_uid).pw_name + group = grp.getgrgid(file_stat.st_gid).gr_name + + # Get file permissions + permissions = stat.filemode(file_stat.st_mode) + + except Exception as e: + raise RuntimeError(f"Error retrieving file info for '{file_path}': {e}") from e + + curr_user = getpass.getuser() + user_info = pwd.getpwnam(curr_user) + group_ids = os.getgrouplist(curr_user, user_info.pw_gid) + curr_groups = [grp.getgrgid(gid).gr_name for gid in group_ids] + + raise PermissionError ("Input file exists but it is not readable for current user\n" + f" - file name: {file_path}\n" + f" - file owner: {owner}\n" + f" - file group: {group}\n" + f" - permissions: {permissions}\n" + f" - current user: {curr_user}\n" + f" - current user groups: {curr_groups}\n") ############################################################################### def do_cime_vars_on_yaml_output_files(case, caseroot): @@ -1017,6 +1048,23 @@ def do_cime_vars_on_yaml_output_files(case, caseroot): print (" - setting skip_t0_output=true\n") print (" - setting freq and freq_units to HIST_N and HIST_OPTION respectively\n") + # If frequency_units is not nsteps, verify that we don't request + # a frequency faster than the model timestep + if content['output_control']['frequency_units'] in ['nsecs','nmins','nhours']: + freq = content['output_control']['Frequency'] + units = content['output_control']['frequency_units'] + dt_out = 1 if units=="nsecs" else 60 if units=="nmins" else 3600 + dt_out = dt_out*int(freq) + + dt_atm = 86400 / case.get_value("ATM_NCPL") + expect (dt_atm<=dt_out, + "Cannot have output frequency faster than atm timestep.\n" + f" yaml file: {fn.strip()}\n" + f" Frequency: {freq}\n" + f" frequency_units: {units}\n" + f" ATM_NCPL: {case.get_value('ATM_NCPL')}\n" + f" This yields dt_atm={dt_atm} > dt_output={dt_out}. Please, adjust 'Frequency' and/or 'frequency_units'\n") + ordered_dump(content, open(dst_yaml, "w")) output_yaml_files.append(dst_yaml) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 50019833d8ae..7e92482e6767 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -48,7 +48,7 @@ be lost if SCREAM_HACK_XML is not enabled. ctl_nl - driver_options,iop_options,atmosphere_processes,grids_manager,initial_conditions,Scorpio,e3sm_parameters + driver_debug_options,driver_options,iop_options,atmosphere_processes,grids_manager,initial_conditions,Scorpio,e3sm_parameters @@ -232,14 +232,19 @@ be lost if SCREAM_HACK_XML is not enabled. 7.0 0.1 0.1 + false 0.001 - 6 + 6 + 0 + + + @@ -260,6 +265,43 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/poly_rrtmg_c20240206.nc + + + + + + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/DMSflux.2010.ne30pg2_conserv.POPmonthlyClimFromACES4BGC_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_bc_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a2_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_num_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_pom_a4_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a1_surf_ne30pg2_2010_clim_c20240816.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/surface/cmip6_mam4_so4_a2_surf_ne30pg2_2010_clim_c20240816.nc + + + + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne1024pg2_20231201.nc + + + + + @@ -342,6 +384,11 @@ be lost if SCREAM_HACK_XML is not enabled. 531.2820e-12 0.7906 1.0e-7 + 1650.e-9 + 348.0e-6 + 306.0e-9 + 0.0 + 0.0 @@ -362,7 +409,7 @@ be lost if SCREAM_HACK_XML is not enabled. 3 3 3 - 3 + 3 4 true false @@ -488,8 +535,10 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0,0.0 - 2.6e-08 - 0.0 + 2.6e-08 + 0.41417721820867320E-007 + 0.15100083211582764E+004 + 0.0 0.0 0.0 0.0 @@ -514,12 +563,21 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0 0.0 + + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 + 1e-5 - 0.0 - 0.0 - 0.0 - 0.0 + 0.0 + 0.0 + 0.0 + 0.0 0.0 @@ -544,6 +602,18 @@ be lost if SCREAM_HACK_XML is not enabled. + + + + + -9999 + + + 0 @@ -572,34 +642,35 @@ be lost if SCREAM_HACK_XML is not enabled. ${CASE} - + true UNSET - ${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc -999 - 31.5 -999 - 238.5 false - true false - true false false 1100 0 10800 false + + + + ${DIN_LOC_ROOT}/atm/cam/scam/iop/RCE_iopfile_4scam.nc + 0.0 + 0.0 + 0 - 2 + 2 False 2 - 1 1 6 0 @@ -613,7 +684,7 @@ be lost if SCREAM_HACK_XML is not enabled. 1 1 3.4e-08 - 0.216784 + 0.216784 UNSET 250000.0 250000.0 @@ -643,13 +714,13 @@ be lost if SCREAM_HACK_XML is not enabled. 1024 0 0 - 5 + 5 0 - 5 + 5 0 - 50000 + 50000 0 - 50000 + 50000 -1 4 cube diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands new file mode 100644 index 000000000000..4cba77f45f81 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/arm97/shell_commands @@ -0,0 +1,15 @@ +# Sets up DPxx case to test ARM97 (continental deep convection) +# case over land using prescribed surface fluxes and nudged winds + +./xmlchange RUN_STARTDATE="1997-06-23" +./xmlchange PTS_LAT=36.605 +./xmlchange PTS_LON=262.515 + +# Scripts location +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +$ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/ARM97_iopfile_4scam.nc -b +$ATMCHANGE target_latitude=36.605 -b +$ATMCHANGE target_longitude=262.515 -b +$ATMCHANGE iop_nudge_uv=true -b +$ATMCHANGE iop_srf_prop=true -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands new file mode 100644 index 000000000000..5b5df60cf822 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/comble/shell_commands @@ -0,0 +1,21 @@ +# Sets up DPxx case to test COMBLE (cold air outbreak) +# lagrangian case over ocean with interactive surface fluxes and coriolis forcing applied to winds + +./xmlchange RUN_STARTDATE="2020-03-12" +./xmlchange START_TOD=79200 +./xmlchange PTS_LAT=74.5 +./xmlchange PTS_LON=9.9 + +./xmlchange SSTICE_DATA_FILENAME='$DIN_LOC_ROOT/ocn/docn7/SSTDATA/sst_HadOIBl_bc_1x1_clim_c101029_COMBLE.nc' +./xmlchange SSTICE_YEAR_ALIGN=2020 +./xmlchange SSTICE_YEAR_START=2020 +./xmlchange SSTICE_YEAR_END=2021 + +# Scripts location +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +$ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/COMBLE_iopfile_4scam.nc -b +$ATMCHANGE target_latitude=74.5 -b +$ATMCHANGE target_longitude=9.9 -b +$ATMCHANGE iop_coriolis=true -b + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands new file mode 100644 index 000000000000..675448efedc6 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/dpxx/dycomsrf01/shell_commands @@ -0,0 +1,15 @@ +# Sets up DPxx case to test DYCOMSrf01 (marine stratocumulus) +# case over ocean using prescribed surface fluxes and compute tendencies of large scale subsidence from prescribed omega + +./xmlchange RUN_STARTDATE="1999-07-10" +./xmlchange PTS_LAT=31.5 +./xmlchange PTS_LON=238.5 + +# Scripts location +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange + +$ATMCHANGE iop_file='${DIN_LOC_ROOT}'/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc -b +$ATMCHANGE target_latitude=31.5 -b +$ATMCHANGE target_longitude=238.5 -b +$ATMCHANGE iop_dosubsidence=true -b +$ATMCHANGE iop_srf_prop=true -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands index cf3ca97f6dd3..f65ccdec105e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands @@ -1,2 +1,2 @@ -$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b ./xmlchange POSTRUN_SCRIPT="$CIMEROOT/../components/eamxx/scripts/check-hashes-ers" diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands index 68eaf51698b3..658c94a3cf67 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands @@ -1,7 +1,9 @@ -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh #modify initial condition file to get aerosol species ICs $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands new file mode 100644 index 000000000000..c5acf055a882 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/drydep/shell_commands @@ -0,0 +1,17 @@ + +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b + +#------------------------------------------------------ +#Update IC file and add drydep process +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_drydep" -b + + + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands index 1c22bd9ee454..224c16586104 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands @@ -1,7 +1,8 @@ - -#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer -#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options -./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh $CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_optics,rrtmgp" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands new file mode 100644 index 000000000000..f47a1729ed74 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/srf_online_emiss_constituent_fluxes/shell_commands @@ -0,0 +1,17 @@ + +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b + +#------------------------------------------------------ +#Update IC file and add the processes +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mam4_constituent_fluxes,mac_aero_mic,rrtmgp,mam4_srf_online_emiss" -b + + + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh new file mode 100755 index 000000000000..c533123847ef --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +#------------------------------------------------------ +# MAM4xx adds additional tracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +# Additional MAM4xx tracers (MAM4xx adds 31 tracers) +ADDITIONAL_MAM4xx_TRACERS=31 + +# Original CMAKE options in env_build.xml +orig_cmake_opt=`./xmlquery --value SCREAM_CMAKE_OPTIONS` + +# Extract the number of tracers +orig_tracer_num=$(echo $orig_cmake_opt | grep -oP 'SCREAM_NUM_TRACERS \K[0-9]+') + +# Update number of tracers +new_tracer_num=$((orig_tracer_num + ADDITIONAL_MAM4xx_TRACERS)) + +# Form the new CMake options string by replacing the original number with the new number +new_cmake_opt=$(echo $orig_cmake_opt | sed "s/SCREAM_NUM_TRACERS $orig_tracer_num/SCREAM_NUM_TRACERS $new_tracer_num/") + +# Update cmake options string +`./xmlchange SCREAM_CMAKE_OPTIONS="$new_cmake_opt"` diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands new file mode 100644 index 000000000000..7b5b9a876503 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/wetscav/shell_commands @@ -0,0 +1,13 @@ +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b + +#------------------------------------------------------ +#Update IC file and add wetscav process +#------------------------------------------------------ +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_wetscav,rrtmgp" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands index 904f46b580f2..54bbf0aba648 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands @@ -29,7 +29,7 @@ $YAML_EDIT_SCRIPT -g \ --avg-type INSTANT \ --freq HIST_N \ --freq-units HIST_OPTION \ - --prefix ${CASE}.scream.diags.hi \ + --prefix ${CASE}.scream.diags.h \ --grid "Physics ${PGTYPE}" \ --fields ${FIELDS} diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands index 116cdf111b43..e327f4d03365 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands @@ -26,7 +26,7 @@ $YAML_EDIT_SCRIPT -g \ --avg-type INSTANT \ --freq HIST_N \ --freq-units HIST_OPTION \ - --prefix ${CASE}.scream.phys.hi \ + --prefix ${CASE}.scream.phys.h \ --grid "Physics ${PGTYPE}" \ --fields ${FIELDS} diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands index 0952b4afe3f9..c20a77d30d5c 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands @@ -26,7 +26,7 @@ $YAML_EDIT_SCRIPT -g \ --avg-type INSTANT \ --freq HIST_N \ --freq-units HIST_OPTION \ - --prefix ${CASE}.scream.phys_dyn.hi \ + --prefix ${CASE}.scream.phys_dyn.h \ --grid Dynamics \ --io-grid 'Physics GLL' \ --fields ${FIELDS} diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands index e6773dce4199..a2678e6f00c0 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands @@ -1,2 +1,2 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_SMALL_KERNELS On' -$CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands new file mode 100644 index 000000000000..a3ec62388f09 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_p3/shell_commands @@ -0,0 +1,2 @@ +./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_P3_SMALL_KERNELS On' +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands new file mode 100644 index 000000000000..ca520c108584 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels_shoc/shell_commands @@ -0,0 +1,2 @@ +./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_SHOC_SMALL_KERNELS On' +$CIMEROOT/../components/eamxx/scripts/atmchange ANY::internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b diff --git a/components/eamxx/cime_config/usermods_dirs/rcemip/user_nl_cpl b/components/eamxx/cime_config/usermods_dirs/rcemip/user_nl_cpl new file mode 100644 index 000000000000..3ecd465f7a14 --- /dev/null +++ b/components/eamxx/cime_config/usermods_dirs/rcemip/user_nl_cpl @@ -0,0 +1,34 @@ +!------------------------------------------------------------------------ +! Users should ONLY USE user_nl_cpl to change namelists variables +! for namelist variables in drv_in (except for the ones below) and +! any keyword/values in seq_maps.rc +! Users should add ALL user specific namelist and seq_maps.rc changes below +! using the following syntax +! namelist_var = new_namelist_value +! or +! mapname = new_map_name +! For example to change the default value of ocn2atm_fmapname to 'foo' use +! ocn2atm_fmapname = 'foo' +! +! Note that some namelist variables MAY NOT be changed in user_nl_cpl - +! they are defined in a $CASEROOT xml file and must be changed with +! xmlchange. +! +! For example, rather than set username to 'foo' in user_nl_cpl, call +! ./xmlchange USER=foo +!------------------------------------------------------------------------ +! +! SPECIAL NOTE FOR AQUAPLANET +! Do not modify any of the following orb_ entries +! co2vmr is not specified via the namelist but rather in the env_run.xml +! +orb_eccen = 0 +orb_obliq = 0 +orb_mvelp = 0 +orb_mode = "fixed_parameters" + +seq_flux_mct_albdif = 0.07 +seq_flux_mct_albdir = 0.07 +seq_flux_atmocn_minwind = 1 + +constant_zenith_deg = 42.05 \ No newline at end of file diff --git a/components/eamxx/cmake/machine-files/anlgce-ub22.cmake b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake new file mode 100644 index 000000000000..e79f70015471 --- /dev/null +++ b/components/eamxx/cmake/machine-files/anlgce-ub22.cmake @@ -0,0 +1,12 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +# Remove this if you are using a resource manager (slurm etc) +set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") + +# EKAT MPI settings +set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") +set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/cmake/machine-files/anlgce.cmake b/components/eamxx/cmake/machine-files/anlgce.cmake index 079000059d19..e79f70015471 100644 --- a/components/eamxx/cmake/machine-files/anlgce.cmake +++ b/components/eamxx/cmake/machine-files/anlgce.cmake @@ -1,12 +1,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) - # Remove this if you are using a resource manager (slurm etc) set (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES True CACHE BOOL "") # EKAT MPI settings set (EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "mpiexec") set (EKAT_MPI_NP_FLAG "-n" CACHE STRING "-n") + +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) +include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake) diff --git a/components/eamxx/cmake/machine-files/dane-intel.cmake b/components/eamxx/cmake/machine-files/dane-intel.cmake new file mode 100644 index 000000000000..df6aad60256b --- /dev/null +++ b/components/eamxx/cmake/machine-files/dane-intel.cmake @@ -0,0 +1,6 @@ +include(${CMAKE_CURRENT_LIST_DIR}/dane.cmake) +set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/mkl/mkl-2022.1.0/lib/intel64/ -qmkl" CACHE STRING "" FORCE) +set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) +option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) +set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) +execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) diff --git a/components/eamxx/cmake/machine-files/dane.cmake b/components/eamxx/cmake/machine-files/dane.cmake new file mode 100644 index 000000000000..aec9f2b42227 --- /dev/null +++ b/components/eamxx/cmake/machine-files/dane.cmake @@ -0,0 +1,13 @@ +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) +common_setup() + +# Load all kokkos settings from Ekat's mach file +set(Kokkos_ENABLE_SERIAL TRUE CACHE BOOL "") + +# Enable Broadwell arch in Kokkos +option(Kokkos_ARCH_BDW "" ON) +# Load Broadwell flags and openmp backend for kokkos +include (${EKAT_MACH_FILES_PATH}/kokkos/intel-bdw.cmake) +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) + +include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) diff --git a/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake b/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake index 14d1d501160d..8704d848334b 100644 --- a/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake +++ b/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake @@ -6,4 +6,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/hip.cmake) set(SCREAM_MPIRUN_EXE "srun" CACHE STRING "") set(SCREAM_MACHINE "frontier-scream-gpu" CACHE STRING "") -set(CMAKE_CXX_FLAGS "--amdgpu-target=gfx90a -fno-gpu-rdc -I$ENV{MPICH_DIR}/include" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS "-Wno-mismatched-tags --offload-arch=gfx90a -munsafe-fp-atomics -fno-gpu-rdc -I$ENV{MPICH_DIR}/include" CACHE STRING "" FORCE) diff --git a/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake b/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake new file mode 100644 index 000000000000..3139580e9ee0 --- /dev/null +++ b/components/eamxx/cmake/machine-files/ghci-snl-openmp.cmake @@ -0,0 +1,9 @@ +# Common settings for our ghci images +include(${CMAKE_CURRENT_LIST_DIR}/ghci-snl.cmake) + +# Set SCREAM_MACHINE +set(SCREAM_MACHINE ghci-snl-openmp CACHE STRING "") + +# Set OpenMP backend +set(EKAT_MACH_FILES_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../../externals/ekat/cmake/machine-files) +include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) diff --git a/components/eamxx/cmake/machine-files/ghci-snl.cmake b/components/eamxx/cmake/machine-files/ghci-snl.cmake new file mode 100644 index 000000000000..f8115aec966b --- /dev/null +++ b/components/eamxx/cmake/machine-files/ghci-snl.cmake @@ -0,0 +1,15 @@ +# Set Fortran flags +set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "Fortran compiler flags" FORCE) + +# Set the path to SCREAM input data +set(SCREAM_INPUT_ROOT /projects/e3sm/inputdata CACHE PATH "Path to SCREAM input data" FORCE) + +# Set the path to BLAS/LAPACK libraries +set(BLAS_LIBRARIES "/spack-installs/netlib-lapack/3.11.0/gcc/12.3.0/base/65x6uge/lib64/libblas.so" CACHE STRING "Path to BLAS library" FORCE) +set(LAPACK_LIBRARIES "/spack-installs/netlib-lapack/3.11.0/gcc/12.3.0/base/65x6uge/lib64/liblapack.so" CACHE STRING "Path to LAPACK library" FORCE) + +# Let's catch usage of code deprecated in Kokkos 4 +option (Kokkos_ENABLE_DEPRECATED_CODE_4 "" OFF) + +# We need to manage resources to spread across available cores/gpus +option (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES "" ON) diff --git a/components/eamxx/cmake/machine-files/mappy.cmake b/components/eamxx/cmake/machine-files/mappy.cmake index 29fb2e74b8a1..976db8223256 100644 --- a/components/eamxx/cmake/machine-files/mappy.cmake +++ b/components/eamxx/cmake/machine-files/mappy.cmake @@ -1,5 +1,4 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -set(PYTHON_EXECUTABLE "/ascldap/users/jgfouca/packages/Python-3.8.5/bin/python3.8" CACHE STRING "" FORCE) set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "" FORCE) diff --git a/components/eamxx/cmake/machine-files/ruby.cmake b/components/eamxx/cmake/machine-files/ruby.cmake index 77d6e6618c71..aec9f2b42227 100644 --- a/components/eamxx/cmake/machine-files/ruby.cmake +++ b/components/eamxx/cmake/machine-files/ruby.cmake @@ -11,5 +11,3 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/intel-bdw.cmake) include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) - -set(SCREAM_INPUT_ROOT "/usr/gdata/e3sm/ccsm3data/inputdata" CACHE STRING "") diff --git a/components/eamxx/docs/user/model_input.md b/components/eamxx/docs/user/model_input.md index a0076c02f89e..def86e25c80f 100644 --- a/components/eamxx/docs/user/model_input.md +++ b/components/eamxx/docs/user/model_input.md @@ -148,7 +148,7 @@ scoped name, which allows `atmchange` to uniquely identify the XML node to modif $ ./atmquery homme::number_of_subcycles namelist_defaults::atmosphere_processes::homme::number_of_subcycles: 1 $ ./atmchange number_of_subcycles=10 -ERROR: number_of_subcycles is ambiguous (use --all to change all matches), matches: +ERROR: internal_diagnostics_level is ambiguous. Use ANY in the node path to allow multiple matches. Matches: namelist_defaults::atmosphere_processes::number_of_subcycles namelist_defaults::atmosphere_processes::sc_import::number_of_subcycles namelist_defaults::atmosphere_processes::homme::number_of_subcycles @@ -157,6 +157,7 @@ ERROR: number_of_subcycles is ambiguous (use --all to change all matches), match namelist_defaults::atmosphere_processes::physics::mac_aero_mic::tms::number_of_subcycles namelist_defaults::atmosphere_processes::physics::mac_aero_mic::shoc::number_of_subcycles namelist_defaults::atmosphere_processes::physics::mac_aero_mic::cldFraction::number_of_subcycles + namelist_defaults::atmosphere_processes::physics::mac_aero_mic::spa::internal_diagnostics_level namelist_defaults::atmosphere_processes::physics::mac_aero_mic::p3::number_of_subcycles namelist_defaults::atmosphere_processes::physics::rrtmgp::number_of_subcycles namelist_defaults::atmosphere_processes::sc_export::number_of_subcycles @@ -167,7 +168,7 @@ $ ./atmquery homme::number_of_subcycles ``` In some cases, the user may be interested in changing _all_ nodes with a given name. In that case, -the `--all` flag can be used: +you can use 'ANY' as a node name: ```shell $ ./atmquery --grep number_of_subcycles @@ -183,7 +184,7 @@ $ ./atmquery --grep number_of_subcycles p3::number_of_subcycles: 1 rrtmgp::number_of_subcycles: 1 sc_export::number_of_subcycles: 1 -$ ./atmchange --all number_of_subcycles=3 +$ ./atmchange ANY::number_of_subcycles=3 Regenerating /path/to/namelist_scream.xml. Manual edits will be lost. $ ./atmquery --grep number_of_subcycles atmosphere_processes::number_of_subcycles: 3 @@ -199,6 +200,24 @@ $ ./atmquery --grep number_of_subcycles rrtmgp::number_of_subcycles: 3 sc_export::number_of_subcycles: 3 ``` +In addition, "ANY" can be used in a "scoped" string, to limit the set of matches: +```shell +$ ./atmchange mac_aero_mic::ANY::number_of_subcycles=1 +Regenerating /path/to/namelist_scream.xml. Manual edits will be lost. +$ ./atmquery --grep number_of_subcycles + atmosphere_processes::number_of_subcycles: 3 + sc_import::number_of_subcycles: 3 + homme::number_of_subcycles: 3 + physics::number_of_subcycles: 3 + mac_aero_mic::number_of_subcycles: 1 + tms::number_of_subcycles: 1 + shoc::number_of_subcycles: 1 + cldFraction::number_of_subcycles: 1 + spa::number_of_subcycles: 1 + p3::number_of_subcycles: 1 + rrtmgp::number_of_subcycles: 3 + sc_export::number_of_subcycles: 3 +``` Since the XML file stores constraints on the parameter value (like its type or valid values), attempting to use the wrong type will cause an error: diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index fbd8381e1727..ae77db7e1826 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -14,7 +14,6 @@ from utils import expect, run_cmd_no_fail ATMCHANGE_SEP = "-ATMCHANGE_SEP-" -ATMCHANGE_ALL = "__ALL__" ATMCHANGE_BUFF_XML_NAME = "SCREAM_ATMCHANGE_BUFFER" ############################################################################### @@ -22,40 +21,32 @@ def apply_atm_procs_list_changes_from_buffer(case, xml): ############################################################################### atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) if atmchg_buffer: - atmchgs, atmchgs_all = unbuffer_changes(case) + atmchgs = unbuffer_changes(case) - expect (len(atmchgs)==len(atmchgs_all),"Failed to unbuffer changes from SCREAM_ATMCHANGE_BUFFER") - for chg, to_all in zip(atmchgs,atmchgs_all): + for chg in atmchgs: if "atm_procs_list" in chg: - expect (not to_all, "Makes no sense to change 'atm_procs_list' for all groups") - atm_config_chg_impl(xml, chg, all_matches=False) + atm_config_chg_impl(xml, chg) ############################################################################### def apply_non_atm_procs_list_changes_from_buffer(case, xml): ############################################################################### atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) if atmchg_buffer: - atmchgs, atmchgs_all = unbuffer_changes(case) + atmchgs = unbuffer_changes(case) - expect (len(atmchgs)==len(atmchgs_all),"Failed to unbuffer changes from SCREAM_ATMCHANGE_BUFFER") - for chg, to_all in zip(atmchgs,atmchgs_all): + for chg in atmchgs: if "atm_procs_list" not in chg: - atm_config_chg_impl(xml, chg, all_matches=to_all) + atm_config_chg_impl(xml, chg) ############################################################################### -def buffer_changes(changes, all_matches=False): +def buffer_changes(changes): ############################################################################### """ Take a list of raw changes and buffer them in the XML case settings. Raw changes are what goes to atm_config_chg_impl. """ # Commas confuse xmlchange and so need to be escaped. - if all_matches: - changes_temp = [c + ATMCHANGE_ALL for c in changes] - changes_str = ATMCHANGE_SEP.join(changes_temp).replace(",",r"\,") - else: - # changes_str += f"{ATMCHANGE_SEP}--all" - changes_str = ATMCHANGE_SEP.join(changes).replace(",",r"\,") + changes_str = ATMCHANGE_SEP.join(changes).replace(",",r"\,") run_cmd_no_fail(f"./xmlchange --append {ATMCHANGE_BUFF_XML_NAME}='{changes_str}{ATMCHANGE_SEP}'") @@ -63,18 +54,15 @@ def buffer_changes(changes, all_matches=False): def unbuffer_changes(case): ############################################################################### """ - From a case, get a list of raw changes. Returns (changes, all_matches_flag) + From a case, get and return a list of raw changes """ atmchg_buffer = case.get_value(ATMCHANGE_BUFF_XML_NAME) atmchgs = [] - atmchgs_all = [] for item in atmchg_buffer.split(ATMCHANGE_SEP): if item.strip(): - atmchgs_all.append(ATMCHANGE_ALL in item) - atmchgs.append(item.replace(ATMCHANGE_ALL,"").replace(r"\,", ",").strip()) - - return atmchgs, atmchgs_all + atmchgs.append(item.replace(r"\,", ",").strip()) + return atmchgs ############################################################################### def reset_buffer(): @@ -101,11 +89,11 @@ def get_xml_nodes(xml_root, name): >>> ################ INVALID SYNTAX ####################### >>> get_xml_nodes(tree,'sub::::prop1') Traceback (most recent call last): - SystemExit: ERROR: Invalid xml node name format, 'sub::::prop1' contains :::: + SystemExit: ERROR: Invalid xml node name format, 'sub::::prop1' contains '::::' >>> ################ VALID USAGE ####################### >>> get_xml_nodes(tree,'invalid::prop1') [] - >>> [item.text for item in get_xml_nodes(tree,'prop1')] + >>> [item.text for item in get_xml_nodes(tree,'ANY::prop1')] ['one', 'two'] >>> [item.text for item in get_xml_nodes(tree,'::prop1')] ['one'] @@ -114,22 +102,66 @@ def get_xml_nodes(xml_root, name): >>> item = get_xml_nodes(tree,'prop2')[0] >>> parent_map = create_parent_map(tree) >>> [p.tag for p in get_parents(item, parent_map)] - ['root', 'sub'] + ['sub'] """ - expect("::::" not in name, f"Invalid xml node name format, '{name}' contains ::::") + expect('::::' not in name, + f"Invalid xml node name format, '{name}' contains '::::'") + + tokens = name.split("::") + expect (tokens[-1] != '', "Input query string ends with '::'. It should end with an actual node name") + if 'ANY' in tokens: + multiple_hits_ok = True + + # Check there's only ONE 'ANY' token + expect(tokens.count('ANY') == 1, "Invalid xml node name format, multiple 'ANY' tokens found.") + + # Split tokens list into two parts: before and after 'ANY' + before_any = tokens[:tokens.index('ANY')] + after_any = tokens[tokens.index('ANY') + 1:] + + # The case where name starts with ::ANY is delicate, since before_any=[''], and this + # trips the call to get_xml_nodes. Since ANY and ::ANY are conceptually the same, + # we set before_any=[] if before_any==[''] + if before_any == ['']: + before_any = [] + + # Call get_xml_nodes with '::'.join(before_any) to get the new root + new_root = get_xml_nodes(xml_root, '::'.join(before_any)) if before_any else [xml_root] + + # Reset xml_root to new_root for the next search + xml_root = new_root[0] + + # Use new_root to find all matches for whatever comes after 'ANY::' + name = '*' if not after_any else '::'.join(after_any) + else: + multiple_hits_ok = False if name.startswith("::"): prefix = "./" # search immediate children only name = name[2:] else: - prefix = ".//" # search entire tree + prefix = ".//" # search entire tree + + # Handle case without ANY try: xpath_str = prefix + name.replace("::", "/") result = xml_root.findall(xpath_str) except SyntaxError as e: expect(False, f"Invalid syntax '{name}' -> {e}") + # Note: don't check that len(result)>0, since user may be ok with 0 matches + if not multiple_hits_ok and len(result)>1: + parent_map = create_parent_map(xml_root) + error_str = f"{name} is ambiguous. Use ANY in the node path to allow multiple matches. Matches:\n" + for node in result: + parents = get_parents(node, parent_map) + name = "::".join(e.tag for e in parents) + name = node.tag if name=="" else name + "::" + node.tag + error_str += " " + name + "\n" + + expect(False, error_str) + return result ############################################################################### @@ -303,7 +335,7 @@ def parse_change(change): return node_name,new_value,append_this ############################################################################### -def atm_config_chg_impl(xml_root, change, all_matches=False): +def atm_config_chg_impl(xml_root, change): ############################################################################### """ >>> xml = ''' @@ -361,9 +393,9 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): >>> ################ AMBIGUOUS CHANGE ####################### >>> atm_config_chg_impl(tree,'prop1=three') Traceback (most recent call last): - SystemExit: ERROR: prop1 is ambiguous (use --all to change all matches), matches: - root::prop1 - root::sub::prop1 + SystemExit: ERROR: prop1 is ambiguous. Use ANY in the node path to allow multiple matches. Matches: + prop1 + sub::prop1 >>> ################ VALID USAGE ####################### >>> atm_config_chg_impl(tree,'::prop1=two') @@ -372,9 +404,9 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): False >>> atm_config_chg_impl(tree,'sub::prop1=one') True - >>> atm_config_chg_impl(tree,'prop1=three', all_matches=True) + >>> atm_config_chg_impl(tree,'ANY::prop1=three') True - >>> [item.text for item in get_xml_nodes(tree,'prop1')] + >>> [item.text for item in get_xml_nodes(tree,'ANY::prop1')] ['three', 'three'] >>> ################ TEST APPEND += ################# >>> atm_config_chg_impl(tree,'a+=4') @@ -413,16 +445,6 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): expect(len(matches) > 0, f"{node_name} did not match any items") - if len(matches) > 1 and not all_matches: - parent_map = create_parent_map(xml_root) - error_str = "" - for node in matches: - parents = get_parents(node, parent_map) - name = "::".join(e.tag for e in parents) + "::" + node.tag - error_str += " " + name + "\n" - - expect(False, f"{node_name} is ambiguous (use --all to change all matches), matches:\n{error_str}") - any_change = False for node in matches: any_change |= apply_change(xml_root, node, new_value, append_this) @@ -432,7 +454,9 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): ############################################################################### def create_parent_map(root): ############################################################################### - return {c: p for p in root.iter() for c in p} + pmap = {c: p for p in root.iter() for c in p} + pmap[root] = None + return pmap ############################################################################### def get_parents(elem, parent_map): @@ -442,25 +466,75 @@ def get_parents(elem, parent_map): be the furthest ancestor, last item will be direct parent) """ results = [] - if elem in parent_map: + if not is_root(elem,parent_map): parent = parent_map[elem] - results = get_parents(parent, parent_map) + [parent] + results = get_parents(parent, parent_map) + if not is_root(parent,parent_map): + results.append(parent) return results +############################################################################### +def is_anchestor_of(parent,child,parent_map): +############################################################################### + """ + >>> xml = ''' + ... + ... one + ... + ... two + ... 2 + ... + ... + ... ''' + >>> import xml.etree.ElementTree as ET + >>> tree = ET.fromstring(xml) + >>> parent_map = create_parent_map(tree) + >>> sub = get_xml_nodes(tree,'sub')[0] + >>> sub_prop1 = get_xml_nodes(tree,'sub::prop1')[0] + >>> is_anchestor_of(sub,sub_prop1,parent_map) + True + >>> is_anchestor_of(sub_prop1,sub,parent_map) + False + """ + curr = child + while curr is not None: + if curr is parent: + return True + curr = parent_map[curr] + return False + +############################################################################### +def is_root (node,parent_map): +############################################################################### + return parent_map[node] is None + ############################################################################### def print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style="invalid",indent=""): ############################################################################### - expect (print_style in ["short","full"], + if len(node)>0: + print (f"{indent}{node.tag}:") + # This is not a leaf, so print all nested nodes. + for child in node: + # Since prints are nicely nested, use 'node-name' as print style + print_var_impl(child,{},full,dtype,value,valid_values,'node-name',indent+" ") + return + + expect (print_style in ["node-name","full-scope","parent-scope"], f"Invalid print_style '{print_style}' for print_var_impl. Use 'full' or 'short'.") - if print_style=="short": + if print_style=="node-name": # Just the inner most name name = node.tag + elif print_style=="parent-scope": + parent = parent_map[node] + name = node.tag if is_root(parent,parent_map) else parent.tag + "::" + node.tag else: parents = get_parents(node, parent_map) - name = "::".join(e.tag for e in parents) + "::" + node.tag + name = "::".join(e.tag for e in parents) + name += "::" if parents else "" + name += node.tag if full: expect ("type" in node.attrib.keys(), @@ -505,59 +579,46 @@ def print_var(xml_root,parent_map,var,full,dtype,value,valid_values,print_style= >>> tree = ET.fromstring(xml) >>> parent_map = create_parent_map(tree) >>> ################ Missing type data ####################### - >>> print_var(tree,parent_map,'::prop1',False,True,False,False,"short") + >>> print_var(tree,parent_map,'::prop1',False,True,False,False,"node-name") Traceback (most recent call last): SystemExit: ERROR: Error! Missing type information for prop1 - >>> print_var(tree,parent_map,'prop2',True,False,False,False,"short") + >>> print_var(tree,parent_map,'prop2',True,False,False,False,"node-name") prop2 value: 2 type: integer valid values: ['1', '2'] - >>> print_var(tree,parent_map,'prop2',False,True,False,False,"short") + >>> print_var(tree,parent_map,'prop2',False,True,False,False,"node-name") prop2: integer - >>> print_var(tree,parent_map,'prop2',False,False,True,False,"short") + >>> print_var(tree,parent_map,'prop2',False,False,True,False,"node-name") 2 - >>> print_var(tree,parent_map,'prop2',False,False,False,True,"short"," ") + >>> print_var(tree,parent_map,'prop2',False,False,False,True,"node-name"," ") prop2: ['1', '2'] """ - expect (print_style in ["short","full"], - f"Invalid print_style '{print_style}' for print_var. Use 'full' or 'short'.") - - # Get the shortest unique repr of the var name - tokens = var.split("::") - if tokens[0]=='': - tokens.pop(0) - - while len(tokens)>1: - new_name = "::".join(tokens[1:]) - matches = get_xml_nodes(xml_root, new_name) - if len(matches) > 1: - break - else: - tokens.pop(0) + expect (print_style in ["node-name","full-scope","parent-scope"], + f"Invalid print_style '{print_style}' for print_var. Use 'node-name', 'full-scope', or 'parent-scope'.") - # Get node, along with all its parents (which might be used for 'full' print style) + # Get matches matches = get_xml_nodes(xml_root,var) - expect(len(matches) == 1, f"Expected one match for {var}") - node = matches[0] - print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style,indent) + # If ANY is in the var name, we may hit the case where one of the matches + # is a parent of another match. In this case, we want to get rid of + unique_matches = [] + for i in matches: + add_this = True + for j in matches: + if i is not j and is_anchestor_of(j,i,parent_map): + add_this = False + + if add_this: + unique_matches.append(i) -############################################################################### -def print_all_vars(xml_root,xml_node,parent_map,curr_namespace,full,dtype,value,valid_values,print_style,indent): -############################################################################### - - print (f"{indent}{xml_node.tag}") - for c in xml_node: - if len(c)>0: - print_all_vars(xml_root,c,parent_map,curr_namespace+c.tag+"::",full,dtype,value,valid_values,print_style,indent+" ") - else: - print_var(xml_root,parent_map,curr_namespace+c.tag,full,dtype,value,valid_values,print_style,indent+" ") + for node in unique_matches: + print_var_impl(node,parent_map,full,dtype,value,valid_values,print_style,indent) ############################################################################### def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, - dtype=False, valid_values=False, grep=False): + dtype=False, valid_values=False, grep=False,parent_map=None): ############################################################################### """ >>> xml = ''' @@ -573,40 +634,45 @@ def atm_query_impl(xml_root,variables,listall=False,full=False,value=False, >>> tree = ET.fromstring(xml) >>> vars = ['prop2','::prop1'] >>> success = atm_query_impl(tree, vars) - root::sub::prop2: 2 - root::prop1: one + sub::prop2: 2 + prop1: one >>> success = atm_query_impl(tree, [], listall=True, valid_values=True) - root + prop1: + sub: prop1: - sub - prop1: - prop2: ['1', '2'] + prop2: ['1', '2'] >>> success = atm_query_impl(tree,['prop1'], grep=True) - root::prop1: one + prop1: one sub::prop1: two """ - parent_map = create_parent_map(xml_root) + + if not parent_map: + parent_map = create_parent_map(xml_root) if listall: - print_all_vars(xml_root,xml_root,parent_map,"::",full,dtype,value,valid_values,"short"," ") + print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") elif grep: for regex in variables: expect("::" not in regex, "query --grep does not support including parent info") var_re = re.compile(f'{regex}') if var_re.search(xml_root.tag): - print_all_vars(xml_root,xml_root,parent_map,"::",full,dtype,value,valid_values,"short"," ") + if len(xml_root)>0: + parents = get_parents(xml_root,parent_map) + print (f"{'::'.join([p.tag for p in parents]) + '::' + xml_root.tag}:") + print_var(xml_root,parent_map,'ANY',full,dtype,value,valid_values,"node-name"," ") else: for elem in xml_root: if len(elem)>0: - atm_query_impl(elem,variables,listall,full,value,dtype,valid_values,grep) + atm_query_impl(elem,variables,listall,full,value,dtype,valid_values,grep,parent_map) else: if var_re.search(elem.tag): nodes = get_xml_nodes(xml_root, "::"+elem.tag) expect(len(nodes) == 1, "No matches?") - print_var_impl(nodes[0],parent_map,full,dtype,value,valid_values,"full"," ") + print_var_impl(nodes[0],parent_map,full,dtype,value,valid_values,"full-scope"," ") else: for var in variables: - print_var(xml_root,parent_map,var,full,dtype,value,valid_values,"full"," ") + pmap = {} if var=='ANY' else parent_map + print_var(xml_root,pmap,var,full,dtype,value,valid_values,"parent-scope"," ") return True diff --git a/components/eamxx/scripts/atmchange b/components/eamxx/scripts/atmchange index ff886ae83507..eb8de06a6931 100755 --- a/components/eamxx/scripts/atmchange +++ b/components/eamxx/scripts/atmchange @@ -23,6 +23,13 @@ sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) from standard_script_setup import * # pylint: disable=wildcard-import from CIME.case import Case +class DeprecatedAllFlag(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + error_msg = f"Error: '{option_string}' has been deprecated and should not be used.\n" + error_msg += " If you want to change ALL matches for a given var, use 'ANY::$varname=value' syntax instead." + print(error_msg, file=sys.stderr) + sys.exit(1) + ############################################################################### def recreate_raw_xml_file(): ############################################################################### @@ -31,7 +38,7 @@ def recreate_raw_xml_file(): create_raw_xml_file(case, caseroot) ############################################################################### -def atm_config_chg(changes, reset=False, all_matches=False, buffer_only=False): +def atm_config_chg(changes, reset=False, buffer_only=False): ############################################################################### if not buffer_only: expect(os.path.exists("namelist_scream.xml"), @@ -66,7 +73,7 @@ def atm_config_chg(changes, reset=False, all_matches=False, buffer_only=False): root = tree.getroot() for change in changes: - this_changed = atm_config_chg_impl(root, change, all_matches) + this_changed = atm_config_chg_impl(root, change) any_change |= this_changed @@ -74,7 +81,7 @@ def atm_config_chg(changes, reset=False, all_matches=False, buffer_only=False): # NOTE: if a change is wrong (e.g., typo in param name), we are still buffering it. # We have no way of checking this, unfortunately. If you get an error that is # not just syntax, your best course of action is to run atmchange --reset. - buffer_changes(changes, all_matches=all_matches) + buffer_changes(changes) if not buffer_only: recreate_raw_xml_file() @@ -96,6 +103,9 @@ OR \033[1;32m# Change param foo to 'hi' (only works if foo is unambiguous)\033[0m > {0} foo=hi + \033[1;32m# Change all matches of param foo to 'hi'\033[0m + > {0} ANY::foo=hi + \033[1;32m# Change params foo to 'hi' and append 'there' to bar (only works if both are unambiguous)\033[0m > {0} foo=hi bar+=there """.format(pathlib.Path(args[0]).name), @@ -103,13 +113,6 @@ OR formatter_class=argparse.ArgumentDefaultsHelpFormatter ) - parser.add_argument( - "-a", "--all", - default=False, - dest="all_matches", - action="store_true", - help="Apply change to all entries matching the name" - ) parser.add_argument( "-r", "--reset", default=False, @@ -117,6 +120,13 @@ OR help="Forget all previous atmchanges", ) + parser.add_argument( + "-a", "--all", + action=DeprecatedAllFlag, + default=argparse.SUPPRESS, + help="This syntax is deprecated and should not be used. Use ANY:: scope instead." + ) + parser.add_argument( "-b", "--buffer-only", default=False, diff --git a/components/eamxx/scripts/atmquery b/components/eamxx/scripts/atmquery index f07ecefa94f1..1dbfbe5f5795 100755 --- a/components/eamxx/scripts/atmquery +++ b/components/eamxx/scripts/atmquery @@ -69,7 +69,7 @@ OR "--grep", default=False, action="store_true", - help="List all matching variables and their values.", + help="List all variables matching the input regex string and their values.", ) parser.add_argument( diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index b128121cf95f..d5ed8e14f2e0 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -63,13 +63,13 @@ class TestBuildnml(unittest.TestCase): return cases[0] if len(cases) == 1 else cases ########################################################################### - def _get_values(self, case, name, value=None, expect_equal=True, all_matches=False): + def _get_values(self, case, name, value=None, expect_equal=True): ########################################################################### """ Queries a name, optionally checking if the value matches or does not match the argument. """ - if not all_matches: + if 'ANY' not in name: orig = run_cmd_assert_result(self, f"./atmquery {name} --value", from_dir=case) if value: if expect_equal: @@ -80,7 +80,7 @@ class TestBuildnml(unittest.TestCase): return [name] else: - output = run_cmd_assert_result(self, f"./atmquery {name} --grep", from_dir=case).splitlines() + output = run_cmd_assert_result(self, f"./atmquery {name}", from_dir=case).splitlines() names = [line.rsplit(": ", maxsplit=1)[0].strip() for line in output] values = [line.rsplit(": ", maxsplit=1)[1].strip() for line in output] @@ -96,17 +96,13 @@ class TestBuildnml(unittest.TestCase): ########################################################################### def _chg_atmconfig(self, changes, case, reset=False, expect_lost=None): ########################################################################### - changes = [(item[0], item[1], False) if len(item) == 2 else item for item in changes] - - expect_lost = reset if expect_lost is None else expect_lost + expect_lost = reset if expect_lost is None else expect_lost changes_unpacked = {} - for name, value, all_matches in changes: - all_matches_opt = "-a" if all_matches else "" - - names = self._get_values(case, name, value=value, expect_equal=False, all_matches=all_matches) + for name, value in changes: + names = self._get_values(case, name, value=value, expect_equal=False) - run_cmd_assert_result(self, f"./atmchange {all_matches_opt} {name}='{value}'", from_dir=case) + run_cmd_assert_result(self, f"./atmchange {name}='{value}'", from_dir=case) for item in names: changes_unpacked[item] = value @@ -123,6 +119,7 @@ class TestBuildnml(unittest.TestCase): def setUp(self): ########################################################################### self._dirs_to_cleanup = [] + self._common_case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") ########################################################################### def tearDown(self): @@ -178,7 +175,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchanges are not lost when eamxx setup is called """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("atm_log_level", "trace"), ("output_to_screen", "true")], case) @@ -196,7 +193,7 @@ class TestBuildnml(unittest.TestCase): """ Test that the append attribute for array-type params in namelist defaults works as expected """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # Add testOnly proc self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa,p3,testOnly")], case) @@ -217,7 +214,7 @@ class TestBuildnml(unittest.TestCase): """ Test that var+=value syntax behaves as expected """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # Append to an existing entry name = 'output_yaml_files' @@ -234,7 +231,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchanges are lost when resetting """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("atm_log_level", "trace")], case, reset=True) @@ -244,7 +241,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchanges are lost if SCREAM_HACK_XML=TRUE """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case run_cmd_assert_result(self, "./xmlchange SCREAM_HACK_XML=TRUE", from_dir=case) @@ -257,7 +254,8 @@ class TestBuildnml(unittest.TestCase): Test that atmchanges via testmod are preserved """ def_mach_comp = \ - run_cmd_assert_result(self, "../CIME/Tools/list_e3sm_tests cime_tiny", from_dir=CIME_SCRIPTS_DIR).splitlines()[-1].split(".")[-1] + run_cmd_assert_result(self, "../CIME/Tools/list_e3sm_tests cime_tiny", from_dir=CIME_SCRIPTS_DIR).splitlines()[-2].split(".")[-1] + case = self._create_test(f"SMS.ne30_ne30.F2010-SCREAMv1.{def_mach_comp}.scream-scream_example_testmod_atmchange --no-build") # Check that the value match what's in the testmod @@ -270,7 +268,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchange works when using 'namespace' syntax foo::bar """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("p3::enable_precondition_checks", "false")], case) @@ -280,7 +278,7 @@ class TestBuildnml(unittest.TestCase): """ Test that atmchange works for array data """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("surf_mom_flux", "40.0,2.0")], case) @@ -288,21 +286,21 @@ class TestBuildnml(unittest.TestCase): def test_atmchanges_on_all_matches(self): ########################################################################### """ - Test that atmchange --all works + Test that atmchange works with ANY """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case - self._chg_atmconfig([("enable_precondition_checks", "false", True)], case) + self._chg_atmconfig([("ANY::enable_precondition_checks", "false")], case) ########################################################################### def test_atmchanges_on_all_matches_plus_spec(self): ########################################################################### """ - Test atmchange --all followed by an atmchange of one of them + Test atmchange with ANY followed by an atmchange of one of them """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case - self._chg_atmconfig([("enable_precondition_checks", "false", True), + self._chg_atmconfig([("ANY::enable_precondition_checks", "false"), ("p3::enable_precondition_checks", "true")], case) ########################################################################### @@ -311,7 +309,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that add atm procs """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa,p3,testOnly")], case) @@ -325,7 +323,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that add atm procs """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # modifying a procs list requires known processes or "_" pre/post suffixes stat, out, err = run_cmd ("./atmchange mac_aero_mic::atm_procs_list=shoc,cldfraction,spa,p3,spiderman", @@ -339,7 +337,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchange does not change buffer if syntax was wrong """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case # Attempting a bad change should not alter the content of SCREAM_ATMCHANGE_BUFFER old = run_cmd_no_fail ("./xmlquery --value SCREAM_ATMCHANGE_BUFFER",from_dir=case) @@ -356,7 +354,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchange errors out with invalid param names """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case stat, out, err = run_cmd ("./atmchange p3::non_existent=3",from_dir=case) expect (stat!=0,"Command './atmchange p3::non_existent=3' should have failed") @@ -367,7 +365,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that add atm proc groups """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case out = run_cmd_no_fail ("./atmchange mac_aero_mic::atm_procs_list=shoc,_my_group_",from_dir=case) @@ -383,7 +381,7 @@ class TestBuildnml(unittest.TestCase): """ Test atmchanges that remove atm procs """ - case = self._create_test("SMS.ne30_ne30.F2010-SCREAMv1 --no-build") + case = self._common_case self._chg_atmconfig([("mac_aero_mic::atm_procs_list", "shoc,cldFraction,spa")], case) diff --git a/components/eamxx/scripts/jenkins/mappy_setup b/components/eamxx/scripts/jenkins/mappy_setup index 8796d5fa60fa..c261cb8a6829 100644 --- a/components/eamxx/scripts/jenkins/mappy_setup +++ b/components/eamxx/scripts/jenkins/mappy_setup @@ -1,8 +1,5 @@ module purge -source /projects/sems/modulefiles/utils/sems-archive-modules-init.sh -module load sems-archive-env -export PATH=/ascldap/users/jgfouca/packages/Python-3.8.5/bin:$PATH -module load sems-archive-git/2.10.1 +module load sems-git/2.42.0 source $JENKINS_SCRIPT_DIR/sandia_son_proxy diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 0b0567c3cce2..f8de73b7d8b8 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -28,25 +28,29 @@ ["mpicxx","mpifort","mpicc"], "bsub -I -q rhel8 -n 4 -gpu num=4", "/home/projects/e3sm/scream/pr-autotester/master-baselines/weaver/"), - "mappy" : (["module purge", "module load sems-archive-env acme-env acme-cmake/3.26.3 acme-gcc/11.2.0 sems-archive-git/2.10.1 acme-openmpi/4.1.4 acme-netcdf/4.7.4/acme", + "mappy" : (["source /projects/sems/modulefiles/utils/sems-modules-init.sh", + "module purge", "module load sems-cmake/3.27.9 sems-git/2.42.0 sems-gcc/11.4.0 sems-openmpi-no-cuda/4.1.6 sems-netcdf-c/4.9.2 sems-netcdf-cxx/4.2 sems-netcdf-fortran/4.6.1 sems-parallel-netcdf/1.12.3 sems-openblas", "export GATOR_INITIAL_MB=4000MB", - "export PATH=/ascldap/users/jgfouca/packages/valgrind-3.22.0/bin:$PATH", ], ["mpicxx","mpifort","mpicc"], "", "/sems-data-store/ACME/baselines/scream/master-baselines"), "lassen" : (["module --force purge", "module load git gcc/8.3.1 cuda/11.8.0 cmake/3.16.8 spectrum-mpi python/3.7.2", "export LLNL_USE_OMPI_VARS='y'", - "export PATH=/usr/gdata/e3sm/netcdf/bin:$PATH", - "export LD_LIBRARY_PATH=/usr/gdata/e3sm/netcdf/lib:$LD_LIBRARY_PATH", + "export PATH=/usr/workspace/e3sm/netcdf/bin:$PATH", + "export LD_LIBRARY_PATH=/usr/workspace/e3sm/netcdf/lib:$LD_LIBRARY_PATH", ], ["mpicxx","mpifort","mpicc"], "bsub -Ip -qpdebug", ""), - "ruby-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "ruby-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), - "quartz-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "dane-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + ["mpicxx","mpifort","mpicc"], + "salloc --partition=pdebug", + ""), + "quartz-intel" : (["module --force purge", "module use --append /usr/workspace/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), @@ -86,9 +90,18 @@ ["mpicxx","mpifort","mpicc"], "", ""), + "anlgce-ub22" : ([". /nfs/gce/software/custom/linux-ubuntu22.04-x86_64/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.2.0/lmod-8.5.6-hkjjxhp/lmod/lmod/init/sh", "module purge", "module load gcc/12.1.0", "export LD_LIBRARY_PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/lib:$LD_LIBRARY_PATH", "export PATH=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/mpich/4.1.2/gcc-12.1.0/bin:/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0/bin:$PATH", "export NetCDF_ROOT=/nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-12.1.0", "export PERL5LIB=/nfs/gce/projects/climate/software/perl5/lib/perl5"], + ["mpicxx","mpifort","mpicc"], + "", + ""), "linux-generic" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-debug" : ([],["mpicxx","mpifort","mpicc"],"", ""), "linux-generic-serial" : ([],["mpicxx","mpifort","mpicc"],"", ""), + "ghci-snl-openmp" : ([], + ["mpicxx","mpifort","mpicc"], + "", + "/projects/e3sm/baselines/scream/master-baselines" + ), } if pathlib.Path("~/.cime/scream_mach_specs.py").expanduser().is_file(): # pylint: disable=no-member @@ -229,9 +242,9 @@ def is_cuda_machine(machine): assert_machine_supported(machine) env_setup_raw = MACHINE_METADATA[machine][0] - env_setup_str = " ".join(env_setup_raw) + env_setup_str = " ".join(env_setup_raw).lower() - return "cuda" in env_setup_str.lower() + return ("no-cuda" not in env_setup_str and "cuda" in env_setup_str) ############################################################################### def setup_mach_env(machine, ctest_j=None): diff --git a/components/eamxx/scripts/test_all_scream.py b/components/eamxx/scripts/test_all_scream.py index 2d1ee9378ff8..61ce833e5640 100644 --- a/components/eamxx/scripts/test_all_scream.py +++ b/components/eamxx/scripts/test_all_scream.py @@ -2,7 +2,7 @@ SharedArea, safe_copy from git_utils import get_current_head, get_current_commit -from test_factory import create_tests, COV +from test_factory import create_tests, COV, CSR from machines_specs import get_mach_compilation_resources, get_mach_testing_resources, \ get_mach_baseline_root_dir, setup_mach_env, is_cuda_machine, \ @@ -386,10 +386,19 @@ def generate_cmake_config(self, test, for_ctest=False): if "SCREAM_DYNAMICS_DYCORE" not in custom_opts_keys: result += " -DSCREAM_DYNAMICS_DYCORE=HOMME" + # For the compute-sanitizer tool 'racecheck', if no option --racecheck-num-workers + # is provided, it will attempt to use all threads available on node. This can cause + # issues if other test cases are being run in parallel. If the option was not specified, + # limit the number of threads availible to racecheck to the number of compile resources. + if self._parallel and isinstance(test, CSR) and not '--racecheck-num-workers' in result: + new_option = ' --racecheck-num-workers=' + str(test.compile_res_count) + index = result.index('--tool=racecheck') + len('--tool=racecheck') + result = result[:index] + new_option + result[index:] + return result ############################################################################### - def get_taskset_range(self, test, for_compile=True): + def get_taskset_resources(self, test, for_compile=True): ############################################################################### res_name = "compile_res_count" if for_compile else "testing_res_count" @@ -410,13 +419,11 @@ def get_taskset_range(self, test, for_compile=True): expect(offset < len(affinity_cp), f"Offset {offset} out of bounds (max={len(affinity_cp)}) for test {test}\naffinity_cp: {affinity_cp}") - start = affinity_cp[offset] - end = start - for i in range(1, getattr(test, res_name)): - expect(affinity_cp[offset+i] == start+i, f"Could not get contiguous range for test {test}") - end = affinity_cp[offset+i] + resources = [] + for i in range(0, getattr(test, res_name)): + resources.append(affinity_cp[offset+i]) - return start, end + return resources ############################################################################### def create_ctest_resource_file(self, test, build_dir): @@ -429,7 +436,7 @@ def create_ctest_resource_file(self, test, build_dir): # res group is where we usually bind an individual MPI rank. # The id of the res groups on is offset so that it is unique across all builds - start, end = self.get_taskset_range(test, for_compile=False) + resources = self.get_taskset_resources(test, for_compile=False) data = {} @@ -439,7 +446,7 @@ def create_ctest_resource_file(self, test, build_dir): # We add leading zeroes to ensure that ids will sort correctly # both alphabetically and numerically devices = [] - for res_id in range(start,end+1): + for res_id in resources: devices.append({"id":f"{res_id:05d}"}) # Add resource groups @@ -448,7 +455,7 @@ def create_ctest_resource_file(self, test, build_dir): with (build_dir/"ctest_resource_file.json").open("w", encoding="utf-8") as outfile: json.dump(data,outfile,indent=2) - return (end-start)+1 + return len(resources) ############################################################################### def generate_ctest_config(self, cmake_config, extra_configs, test): @@ -489,11 +496,11 @@ def generate_ctest_config(self, cmake_config, extra_configs, test): # Ctest can only competently manage test pinning across a single instance of ctest. For # multiple concurrent instances of ctest, we have to help it. It's OK to use the compile_res_count - # taskset range even though the ctest script is also running the tests + # taskset resources even though the ctest script is also running the tests if self._parallel: - start, end = self.get_taskset_range(test) + resources = self.get_taskset_resources(test) result = result.replace("'", r"'\''") # handle nested quoting - result = f"taskset -c {start}-{end} sh -c '{result}'" + result = f"taskset -c {','.join([str(r) for r in resources])} sh -c '{result}'" return result @@ -530,8 +537,8 @@ def generate_baselines(self, test): cmd = f"make -j{test.compile_res_count}" if self._parallel: - start, end = self.get_taskset_range(test) - cmd = f"taskset -c {start}-{end} sh -c '{cmd}'" + resources = self.get_taskset_resources(test) + cmd = f"taskset -c {','.join([str(r) for r in resources])} sh -c '{cmd}'" stat, _, err = run_cmd(cmd, from_dir=test_dir, verbose=True) diff --git a/components/eamxx/scripts/test_factory.py b/components/eamxx/scripts/test_factory.py index 0b3f0c738c88..f85461ad23b4 100644 --- a/components/eamxx/scripts/test_factory.py +++ b/components/eamxx/scripts/test_factory.py @@ -176,7 +176,7 @@ def __init__(self, _): "debug with compute sanitizer memcheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=memcheck")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=memcheck'")], uses_baselines=False, on_by_default=False, default_test_len="short" @@ -210,7 +210,7 @@ def __init__(self, _): "debug with compute sanitizer initcheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=initcheck")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=initcheck'")], uses_baselines=False, on_by_default=False, default_test_len="short" @@ -227,7 +227,7 @@ def __init__(self, _): "debug with compute sanitizer synccheck", [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=synccheck")], + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=synccheck'")], uses_baselines=False, on_by_default=False, default_test_len="short" diff --git a/components/eamxx/src/CMakeLists.txt b/components/eamxx/src/CMakeLists.txt index 7672d79f13d6..be88a2f51ae0 100644 --- a/components/eamxx/src/CMakeLists.txt +++ b/components/eamxx/src/CMakeLists.txt @@ -8,6 +8,6 @@ if (PROJECT_NAME STREQUAL "E3SM") add_subdirectory(mct_coupling) endif() -if (EAMXX_ENABLE_PYBIND) +if (EAMXX_ENABLE_PYSCREAM) add_subdirectory(python) endif() diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index c3da1bacd3f0..a4bd4fabc528 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -300,6 +300,9 @@ void AtmosphereDriver::create_grids() void AtmosphereDriver::setup_surface_coupling_data_manager(SurfaceCouplingTransferType transfer_type, const int num_cpl_fields, const int num_scream_fields, const int field_size, Real* data_ptr, +#ifdef HAVE_MOAB + Real* data_ptr_moab, +#endif char* names_ptr, int* cpl_indices_ptr, int* vec_comps_ptr, Real* constant_multiple_ptr, bool* do_transfer_during_init_ptr) { @@ -318,6 +321,9 @@ void AtmosphereDriver::setup_surface_coupling_data_manager(SurfaceCouplingTransf } else EKAT_ERROR_MSG("Error! Unexpected SurfaceCouplingTransferType."); sc_data_mgr->setup_internals(num_cpl_fields, num_scream_fields, field_size, data_ptr, +#ifdef HAVE_MOAB + data_ptr_moab, +#endif names_ptr, cpl_indices_ptr, vec_comps_ptr, constant_multiple_ptr, do_transfer_during_init_ptr); } @@ -1588,6 +1594,13 @@ initialize (const ekat::Comm& atm_comm, void AtmosphereDriver::run (const int dt) { start_timer("EAMxx::run"); + // DEBUG option: Check if user has set the run to fail at a specific timestep. + auto& debug = m_atm_params.sublist("driver_debug_options"); + auto fail_step = debug.get("force_crash_nsteps",-1); + if (fail_step==m_current_ts.get_num_steps()) { + std::abort(); + } + // Make sure the end of the time step is after the current start_time EKAT_REQUIRE_MSG (dt>0, "Error! Input time step must be positive.\n"); diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index d43d233f4e97..7e96225210db 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -87,6 +87,9 @@ class AtmosphereDriver void setup_surface_coupling_data_manager(SurfaceCouplingTransferType transfer_type, const int num_cpl_fields, const int num_scream_fields, const int field_size, Real* data_ptr, +#ifdef HAVE_MOAB + Real* data_ptr_moab, +#endif char* names_ptr, int* cpl_indices_ptr, int* vec_comps_ptr, Real* constant_multiple_ptr, bool* do_transfer_during_init_ptr); diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp index 7a0253ec908a..9044df7e76cc 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.cpp @@ -137,6 +137,13 @@ void SurfaceCouplingExporter::setup_surface_coupling_data(const SCDataManager &s m_num_cols, m_num_cpl_exports); m_cpl_exports_view_d = Kokkos::create_mirror_view(DefaultDevice(), m_cpl_exports_view_h); +#ifdef HAVE_MOAB + // The export data is of size num_cpl_exports,ncols. All other data is of size num_scream_exports + m_moab_cpl_exports_view_h = decltype(m_moab_cpl_exports_view_h) (sc_data_manager.get_field_data_moab_ptr(), + m_num_cpl_exports, m_num_cols); + m_moab_cpl_exports_view_d = Kokkos::create_mirror_view(DefaultDevice(), m_moab_cpl_exports_view_h); +#endif + m_export_field_names = new name_t[m_num_scream_exports]; std::memcpy(m_export_field_names, sc_data_manager.get_field_name_ptr(), m_num_scream_exports*32*sizeof(char)); @@ -536,6 +543,12 @@ void SurfaceCouplingExporter::do_export_to_cpl(const bool called_during_initiali // Any field not exported by scream, or not exported // during initialization, is set to 0.0 Kokkos::deep_copy(m_cpl_exports_view_d, 0.0); +#ifdef HAVE_MOAB + // Any field not exported by scream, or not exported + // during initialization, is set to 0.0 + Kokkos::deep_copy(m_moab_cpl_exports_view_d, 0.0); + const auto moab_cpl_exports_view_d = m_moab_cpl_exports_view_d; +#endif const auto cpl_exports_view_d = m_cpl_exports_view_d; const int num_exports = m_num_scream_exports; const int num_cols = m_num_cols; @@ -554,9 +567,27 @@ void SurfaceCouplingExporter::do_export_to_cpl(const bool called_during_initiali cpl_exports_view_d(icol,info.cpl_indx) = info.constant_multiple*info.data[offset]; } }); +#ifdef HAVE_MOAB + Kokkos::parallel_for(export_policy, KOKKOS_LAMBDA(const int& i) { + const int ifield = i / num_cols; + const int icol = i % num_cols; + const auto& info = col_info(ifield); + const auto offset = icol*info.col_stride + info.col_offset; + // if this is during initialization, check whether or not the field should be exported + bool do_export = (not called_during_initialization || info.transfer_during_initialization); + if (do_export) { + moab_cpl_exports_view_d(info.cpl_indx, icol) = info.constant_multiple*info.data[offset]; + } + }); +#endif // Deep copy fields from device to cpl host array Kokkos::deep_copy(m_cpl_exports_view_h,m_cpl_exports_view_d); +#ifdef HAVE_MOAB + // Deep copy fields from device to cpl host array + Kokkos::deep_copy(m_moab_cpl_exports_view_h,m_moab_cpl_exports_view_d); +#endif + } // ========================================================================================= void SurfaceCouplingExporter::finalize_impl() diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp index d8ccf5862f3c..ca7772fdf1be 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_exporter.hpp @@ -141,6 +141,13 @@ class SurfaceCouplingExporter : public AtmosphereProcess view_2d m_cpl_exports_view_d; uview_2d m_cpl_exports_view_h; +#ifdef HAVE_MOAB + // Views storing a 2d array with dims (num_fields, num_cols) for moab cpl export data. + // The field cols strides faster, since that's what moab does (so we can "view" the + // pointer to the whole a2x_am(:,:) array from Fortran) + view_2d m_moab_cpl_exports_view_d; + uview_2d m_moab_cpl_exports_view_h; +#endif // Array storing the field names for exports name_t* m_export_field_names; std::vector m_export_field_names_vector; diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 52deba16e8d3..ee3e21e7461b 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -52,6 +52,11 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptr("snow_depth_land", scalar2d_layout, m, grid_name); add_field("ocnfrac", scalar2d_layout, nondim, grid_name); add_field("landfrac", scalar2d_layout, nondim, grid_name); + add_field("icefrac", scalar2d_layout, nondim, grid_name); + // Friction velocity [m/s] + add_field("fv", scalar2d_layout, m/s, grid_name); + // Aerodynamical resistance + add_field("ram1", scalar2d_layout, s/m, grid_name); } // ========================================================================================= void SurfaceCouplingImporter::setup_surface_coupling_data(const SCDataManager &sc_data_manager) @@ -69,6 +74,13 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptrget_params().get("iop_srf_prop")) { // Overwrite imports with data from IOP file @@ -227,6 +263,9 @@ void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_in const auto& info_d = col_info_d(ifield); const auto offset = icol*info_d.col_stride + info_d.col_offset; info_d.data[offset] = col_val; +#ifdef HAVE_MOAB + // TODO +#endif }); } } diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp index 3a34a8b2951e..6cf04c4166d2 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp @@ -83,6 +83,14 @@ class SurfaceCouplingImporter : public AtmosphereProcess view_2d m_cpl_imports_view_d; uview_2d m_cpl_imports_view_h; +#ifdef HAVE_MOAB + // Views storing a 2d array with dims (num_fields,num_cols) for import data. + // The colums index strides faster, since that's what moab does (so we can "view" the + // pointer to the whole x2a_am(:,:) array from Fortran) + view_2d m_moab_cpl_imports_view_d; + uview_2d m_moab_cpl_imports_view_h; +#endif + // Array storing the field names for imports name_t* m_import_field_names; diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index a7a1ee9eaeaf..be51f4346155 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -20,6 +20,7 @@ set(DIAGNOSTIC_SRCS aodvis.cpp number_path.cpp aerocom_cld.cpp + atm_backtend.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) diff --git a/components/eamxx/src/diagnostics/aodvis.cpp b/components/eamxx/src/diagnostics/aodvis.cpp index efb9b5ba9510..ac16c0033730 100644 --- a/components/eamxx/src/diagnostics/aodvis.cpp +++ b/components/eamxx/src/diagnostics/aodvis.cpp @@ -2,6 +2,8 @@ #include +#include "share/util/scream_universal_constants.hpp" + namespace scream { AODVis::AODVis(const ekat::Comm &comm, const ekat::ParameterList ¶ms) @@ -23,11 +25,13 @@ void AODVis::set_grids( m_nlevs = grid->get_num_vertical_levels(); // Define layouts we need (both inputs and outputs) - FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,m_swbands,"swband"); - FieldLayout scalar1d_layout = grid->get_2d_scalar_layout(); + FieldLayout scalar3d_swband_layout = + grid->get_3d_vector_layout(true, m_swbands, "swband"); + FieldLayout scalar1d_layout = grid->get_2d_scalar_layout(); // The fields required for this diagnostic to be computed add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); + add_field("sunlit", scalar1d_layout, nondim, grid_name); // Construct and allocate the aodvis field FieldIdentifier fid("AerosolOpticalDepth550nm", scalar1d_layout, nondim, @@ -41,18 +45,25 @@ void AODVis::compute_diagnostic_impl() { using MT = typename KT::MemberType; using ESU = ekat::ExeSpaceUtils; + Real var_fill_value = constants::DefaultFillValue().value; + const auto aod = m_diagnostic_output.get_view(); const auto tau_vis = get_field_in("aero_tau_sw") .subfield(1, m_vis_bnd) .get_view(); + const auto sunlit = get_field_in("sunlit").get_view(); const auto num_levs = m_nlevs; const auto policy = ESU::get_default_team_policy(m_ncols, m_nlevs); Kokkos::parallel_for( "Compute " + name(), policy, KOKKOS_LAMBDA(const MT &team) { const int icol = team.league_rank(); - auto tau_icol = ekat::subview(tau_vis, icol); - aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); + if(sunlit(icol) == 0.0) { + aod(icol) = var_fill_value; + } else { + auto tau_icol = ekat::subview(tau_vis, icol); + aod(icol) = ESU::view_reduction(team, 0, num_levs, tau_icol); + } }); } diff --git a/components/eamxx/src/diagnostics/atm_backtend.cpp b/components/eamxx/src/diagnostics/atm_backtend.cpp new file mode 100644 index 000000000000..4b147de2d6c8 --- /dev/null +++ b/components/eamxx/src/diagnostics/atm_backtend.cpp @@ -0,0 +1,88 @@ +#include "diagnostics/atm_backtend.hpp" + +#include + +#include "share/util/scream_universal_constants.hpp" + +namespace scream { + +AtmBackTendDiag::AtmBackTendDiag(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereDiagnostic(comm, params) { + EKAT_REQUIRE_MSG(params.isParameter("Tendency Name"), + "Error! AtmBackTendDiag requires 'Tendency Name' in its " + "input parameters.\n"); + + m_name = m_params.get("Tendency Name"); +} + +std::string AtmBackTendDiag::name() const { return m_name + "_atm_tend"; } + +void AtmBackTendDiag::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + + const auto &gname = m_params.get("grid_name"); + add_field(m_name, gname); +} + +void AtmBackTendDiag::initialize_impl(const RunType /*run_type*/) { + const auto &f = get_field_in(m_name); + const auto &fid = f.get_header().get_identifier(); + const auto &gn = fid.get_grid_name(); + + // Sanity checks + using namespace ShortFieldTagsNames; + const auto &layout = fid.get_layout(); + EKAT_REQUIRE_MSG( + f.data_type() == DataType::RealType, + "Error! AtmBackTendDiag only supports Real data type field.\n" + " - field name: " + + fid.name() + + "\n" + " - field data type: " + + e2str(f.data_type()) + "\n"); + + using namespace ekat::units; + // The units are the same except per second + auto diag_units = fid.get_units() / s; + + // All good, create the diag output + FieldIdentifier d_fid(name(), layout.clone(), diag_units, gn); + m_diagnostic_output = Field(d_fid); + m_diagnostic_output.allocate_view(); + + // Let's also create the previous field + FieldIdentifier prev_fid(name() + "_prev", layout.clone(), diag_units, gn); + m_f_prev = Field(prev_fid); + m_f_prev.allocate_view(); +} + +void AtmBackTendDiag::init_timestep(const util::TimeStamp &start_of_step) { + const auto &f_curr = get_field_in(m_name); + m_f_prev.deep_copy(f_curr); + m_f_prev.get_header().get_tracking().update_time_stamp(start_of_step); +} + +void AtmBackTendDiag::compute_diagnostic_impl() { + Real var_fill_value = constants::DefaultFillValue().value; + std::int64_t dt; + + const auto &f = get_field_in(m_name); + const auto &curr_ts = f.get_header().get_tracking().get_time_stamp(); + const auto &prev_ts = m_f_prev.get_header().get_tracking().get_time_stamp(); + + if(prev_ts.is_valid()) { + // This diag was called before, so we have a valid value for m_f_prev, + // and can compute the tendency + dt = curr_ts - prev_ts; + m_f_prev.update(f, 1.0 / dt, -1.0 / dt); + m_diagnostic_output.deep_copy(m_f_prev); + } else { + // This is the first time we evaluate this diag. We cannot compute a tend + // yet, so fill with an invalid value + m_diagnostic_output.deep_copy(var_fill_value); + } +} + +} // namespace scream diff --git a/components/eamxx/src/diagnostics/atm_backtend.hpp b/components/eamxx/src/diagnostics/atm_backtend.hpp new file mode 100644 index 000000000000..a8bd72883dbe --- /dev/null +++ b/components/eamxx/src/diagnostics/atm_backtend.hpp @@ -0,0 +1,50 @@ +#ifndef EAMXX_ATM_BACKTEND_DIAG_HPP +#define EAMXX_ATM_BACKTEND_DIAG_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" +#include "share/util/scream_time_stamp.hpp" + +namespace scream { + +/* + * This diagnostic will back out the atmosphere tendency of a given field. + */ + +class AtmBackTendDiag : public AtmosphereDiagnostic { + public: + // Constructors + AtmBackTendDiag(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The name of the diagnostic + std::string name() const; + + // Set the grid + void set_grids(const std::shared_ptr grids_manager); + + protected: +#ifdef KOKKOS_ENABLE_CUDA + public: +#endif + void compute_diagnostic_impl(); + + // Let's override the init time step method + void init_timestep(const util::TimeStamp &start_of_step) override; + + // Let's override the initialize method to set the fields below + void initialize_impl(const RunType /*run_type*/) override; + + // Keep track of field dimensions + int m_num_cols; + int m_num_levs; + + // The tendency of what? + std::string m_name; + + // Store the previous field + Field m_f_prev; + +}; // class AtmBackTendDiag + +} // namespace scream + +#endif // EAMXX_ATM_BACKTEND_DIAG_HPP diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index 21d82b045dab..45a04eebeb1f 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -23,6 +23,7 @@ #include "diagnostics/aodvis.hpp" #include "diagnostics/number_path.hpp" #include "diagnostics/aerocom_cld.hpp" +#include "diagnostics/atm_backtend.hpp" namespace scream { @@ -55,6 +56,7 @@ inline void register_diagnostics () { diag_factory.register_product("AerosolOpticalDepth550nm",&create_atmosphere_diagnostic); diag_factory.register_product("NumberPath",&create_atmosphere_diagnostic); diag_factory.register_product("AeroComCld",&create_atmosphere_diagnostic); + diag_factory.register_product("AtmBackTendDiag",&create_atmosphere_diagnostic); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index 2ea6bcffd88a..a684aa248fc0 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -71,4 +71,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) # Test AEROCOM_CLD CreateDiagTest(aerocom_cld "aerocom_cld_test.cpp") + # Test atm_tend + CreateDiagTest(atm_backtend "atm_backtend_test.cpp") + endif() diff --git a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp index 140114812215..8090a9300fd9 100644 --- a/components/eamxx/src/diagnostics/tests/aodvis_test.cpp +++ b/components/eamxx/src/diagnostics/tests/aodvis_test.cpp @@ -1,10 +1,11 @@ +#include + #include "catch2/catch.hpp" #include "diagnostics/register_diagnostics.hpp" #include "share/field/field_utils.hpp" #include "share/grid/mesh_free_grids_manager.hpp" #include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_utils.hpp" - +#include "share/util/scream_universal_constants.hpp" namespace scream { std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, @@ -30,6 +31,10 @@ TEST_CASE("aodvis") { using namespace ShortFieldTagsNames; using namespace ekat::units; + Real var_fill_value = constants::DefaultFillValue().value; + + Real some_limit = 0.0025; + // A world comm ekat::Comm comm(MPI_COMM_WORLD); @@ -40,7 +45,7 @@ TEST_CASE("aodvis") { // Create a grids manager - single column for these tests constexpr int nlevs = 33; - const int ngcols = 2 * comm.size(); + const int ngcols = 10 * comm.size(); int nbnds = eamxx_swbands(); int swvis = eamxx_vis_swband_idx(); @@ -49,16 +54,23 @@ TEST_CASE("aodvis") { auto grid = gm->get_grid("Physics"); // Input (randomized) tau - FieldLayout scalar3d_swband_layout = grid->get_3d_vector_layout(true,nbnds,"swband"); + FieldLayout scalar3d_swband_layout = + grid->get_3d_vector_layout(true, nbnds, "swband"); FieldIdentifier tau_fid("aero_tau_sw", scalar3d_swband_layout, nondim, grid->name()); Field tau(tau_fid); tau.allocate_view(); tau.get_header().get_tracking().update_time_stamp(t0); + // Input (randomized) sunlit + FieldLayout scalar2d_layout = grid->get_2d_scalar_layout(); + FieldIdentifier sunlit_fid("sunlit", scalar2d_layout, nondim, grid->name()); + Field sunlit(sunlit_fid); + sunlit.allocate_view(); + sunlit.get_header().get_tracking().update_time_stamp(t0); // Construct random number generator stuff using RPDF = std::uniform_real_distribution; - RPDF pdf(0, 0.05); + RPDF pdf(0, 0.005); auto engine = scream::setup_random_test(); // Construct the Diagnostics @@ -71,13 +83,27 @@ TEST_CASE("aodvis") { // Randomize tau randomize(tau, engine, pdf); + // Randomize sunlit + randomize(sunlit, engine, pdf); + // Create and set up the diagnostic ekat::ParameterList params; auto diag = diag_factory.create("AerosolOpticalDepth550nm", comm, params); diag->set_grids(gm); diag->set_required_field(tau); + diag->set_required_field(sunlit); diag->initialize(t0, RunType::Initial); + auto sun_h = sunlit.get_view(); + for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { + // zero out all sun_h if below 0.05 + if(sun_h(icol) < some_limit) { + sun_h(icol) = 0.0; + } + } + // sync to device + sunlit.sync_to_dev(); + // Run diag diag->compute_diagnostic(); @@ -93,12 +119,17 @@ TEST_CASE("aodvis") { auto aod_t = aod_tf.get_view(); for(int icol = 0; icol < grid->get_num_local_dofs(); ++icol) { - for(int ilev = 0; ilev < nlevs; ++ilev) { - aod_t(icol) += tau_h(icol, swvis, ilev); + if(sun_h(icol) < some_limit) { + aod_t(icol) = var_fill_value; + } else { + for(int ilev = 0; ilev < nlevs; ++ilev) { + aod_t(icol) += tau_h(icol, swvis, ilev); + } } } aod_hf.sync_to_dev(); aod_tf.sync_to_dev(); + // Workaround for non-bfb behavior of view_reduction() in release builds if(SCREAM_BFB_TESTING) { REQUIRE(views_are_equal(aod_hf, aod_tf)); diff --git a/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp b/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp new file mode 100644 index 000000000000..47086b5ddedc --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/atm_backtend_test.cpp @@ -0,0 +1,114 @@ +#include "catch2/catch.hpp" +#include "diagnostics/register_diagnostics.hpp" +#include "share/field/field_utils.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_universal_constants.hpp" + +namespace scream { + +std::shared_ptr create_gm(const ekat::Comm &comm, const int ncols, + const int nlevs) { + const int num_global_cols = ncols * comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names", vos_t{"Point Grid"}); + auto &pl = gm_params.sublist("Point Grid"); + pl.set("type", "point_grid"); + pl.set("aliases", vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm, gm_params); + gm->build_grids(); + + return gm; +} + +TEST_CASE("atm_backtend") { + using namespace ShortFieldTagsNames; + using namespace ekat::units; + + // A world comm + ekat::Comm comm(MPI_COMM_WORLD); + + // A time stamp + util::TimeStamp t0({2024, 1, 1}, {0, 0, 0}); + + // Create a grids manager - single column for these tests + constexpr int nlevs = 7; + const int ngcols = 2 * comm.size(); + + auto gm = create_gm(comm, ngcols, nlevs); + auto grid = gm->get_grid("Physics"); + + // Input (randomized) qc + FieldLayout scalar2d_layout{{COL, LEV}, {ngcols, nlevs}}; + FieldIdentifier qc_fid("qc", scalar2d_layout, kg / kg, grid->name()); + + Field qc(qc_fid); + qc.allocate_view(); + + // Construct random number generator stuff + using RPDF = std::uniform_real_distribution; + RPDF pdf(0.0, 200.0); + + auto engine = scream::setup_random_test(); + + // Construct the Diagnostics + std::map> diags; + auto &diag_factory = AtmosphereDiagnosticFactory::instance(); + register_diagnostics(); + + ekat::ParameterList params; + REQUIRE_THROWS(diag_factory.create("AtmBackTendDiag", comm, + params)); // No 'Tendency Name' + + Real var_fill_value = constants::DefaultFillValue().value; + + // Set time for qc and randomize its values + qc.get_header().get_tracking().update_time_stamp(t0); + randomize(qc, engine, pdf); + + // Create and set up the diagnostic + params.set("grid_name", grid->name()); + params.set("Tendency Name", "qc"); + auto diag = diag_factory.create("AtmBackTendDiag", comm, params); + diag->set_grids(gm); + diag->set_required_field(qc); + diag->initialize(t0, RunType::Initial); + + // Run diag + diag->compute_diagnostic(); + auto diag_f = diag->get_diagnostic(); + + // Check result: diag should be filled with var_fill_value + auto some_field = qc.clone(); + some_field.deep_copy(var_fill_value); + REQUIRE(views_are_equal(diag_f, some_field)); + + const Real a_day = 24.0 * 60.0 * 60.0; // seconds + + constexpr int ntests = 10; + + for(int itest = 2; itest < ntests; itest++) { + // Run diag again + some_field.deep_copy(qc); + + diag->init_timestep(t0); + + util::TimeStamp t1({2024, 1, itest}, {0, 0, 0}); // a day later + qc.get_header().get_tracking().update_time_stamp(t1); + randomize(qc, engine, pdf); + + diag->compute_diagnostic(); + some_field.update(qc, 1.0 / a_day, -1.0 / a_day); + REQUIRE(views_are_equal(diag_f, some_field)); + + // reset t0 to t1 to keep iterating... + t0 = t1; + } +} + +} // namespace scream diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp index eaf65a69cda5..cf70dd49f564 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -10,14 +10,14 @@ // Homme includes #include "Context.hpp" #include "ColumnOps.hpp" -#include "ElementOps.hpp" -#include "EquationOfState.hpp" #include "HommexxEnums.hpp" #include "HybridVCoord.hpp" -#include "KernelVariables.hpp" #include "SimulationParams.hpp" #include "Types.hpp" +// SCREAM includes +#include "share/util/scream_common_physics_functions.hpp" + // EKAT includes #include "ekat/ekat_workspace.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" @@ -222,11 +222,7 @@ void HommeDynamics:: apply_iop_forcing(const Real dt) { using ESU = ekat::ExeSpaceUtils; - - using EOS = Homme::EquationOfState; - using ElementOps = Homme::ElementOps; - using KV = Homme::KernelVariables; - + using PF = PhysicsFunctions; using ColOps = ColumnOps; using C = physics::Constants; constexpr Real Rair = C::Rair; @@ -263,7 +259,7 @@ apply_iop_forcing(const Real dt) const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); - // Homme element states and EOS/EO classes + // Homme element states auto ps_dyn = get_internal_field("ps_dyn").get_view(); auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); @@ -272,13 +268,6 @@ apply_iop_forcing(const Real dt) auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); - EOS eos; - eos.init(params.theta_hydrostatic_mode, hvcoord); - - ElementOps elem_ops; - elem_ops.init(hvcoord); - const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); - // Load data from IOP files, if necessary m_iop->read_iop_file_data(timestamp()); @@ -319,92 +308,66 @@ apply_iop_forcing(const Real dt) : m_iop->get_iop_field("v").get_view(); } - // Team policy and workspace manager for both homme and scream - // related loops. We need separate policies since hommexx functions used here - // assume they are called inside nested loops for elements and Gaussian points, - // whereas EAMxx function we use expects a single level of parallelism - // for elements and Guassian points. - // TODO: scream::ColumnOps functions could take an arbitary loop boundary - // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that - // all 3 kernel launches here could be combined. - const auto policy_homme = ESU::get_default_team_policy(nelem, NLEV); - const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); + // Team policy and workspace manager for eamxx + const auto policy_iop = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); // TODO: Create a memory buffer for this class // and add the below WSM and views - WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); - WorkspaceMgr homme_wsm(NLEV, 16 + (theta_hydrostatic_mode ? 16 : 0), policy_homme); + WorkspaceMgr iop_wsm(NLEVI, 7+qsize, policy_iop); view_Nd - rstar ("rstar", nelem, NGP, NGP, NLEV), - exner ("exner", nelem, NGP, NGP, NLEV), temperature("temperature", nelem, NGP, NGP, NLEV); - // Lambda for computing rstar, exner, and temperature from Hommexx - auto compute_homme_states = [&] () { - Kokkos::parallel_for("compute_rstar_exner_and_temperature", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + // Lambda for computing temperature + auto compute_temperature = [&] () { + Kokkos::parallel_for("compute_temperature_for_iop", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = homme_wsm.get_workspace(team); - auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); - uview_2d pnh(reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto pnh_i = ekat::subview(pnh, idx); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); - Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); - Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); - Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); - Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); - - // Compute exner from EOS - if (theta_hydrostatic_mode) { - auto hydro_p_int = ws.take("hydro_p_int"); - Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); - elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); - eos.compute_exner(kv, pnh_scalar, exner_scalar); - ws.release(hydro_p_int); - } else { - eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); - } - - // Get the temperature from dynamics states - elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid; + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); + + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + // Compute temperature from virtual potential temperature + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { + auto T_val = vtheta_dp_i(k); + T_val /= dp3d_i(k); + T_val = PF::calculate_temperature_from_virtual_temperature(T_val,qv_i(k)); + temperature_i(k) = PF::calculate_T_from_theta(T_val,pmid(k)); }); // Release WS views - ws.release_macro_block(pnh_slot, NGP*NGP); + ws.release_many_contiguous<1>({&pmid}); }); }; - // Preprocess some homme states to get temperature and exner - compute_homme_states(); + // Preprocess some homme states to get temperature + compute_temperature(); Kokkos::fence(); // Apply IOP forcing - Kokkos::parallel_for("apply_iop_forcing", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + Kokkos::parallel_for("apply_iop_forcing", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank()/(NGP*NGP); const int igp = (team.league_rank()/NGP)%NGP; const int jgp = team.league_rank()%NGP; // Get temp views from workspace - auto ws = eamxx_wsm.get_workspace(team); + auto ws = iop_wsm.get_workspace(team); uview_1d pmid, pint, pdel; ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, {&pmid, &pint, &pdel}); @@ -449,44 +412,57 @@ apply_iop_forcing(const Real dt) Kokkos::fence(); // Postprocess homme states Qdp and vtheta_dp - Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; + // Get temp views from workspace + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); - // Reinterperate into views of Homme::Scalar for calling Hommexx function. - Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); - Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + team.team_barrier(); - // Compute Qdp from updated Q - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { - const int ilev = k/qsize; - const int q = k%qsize; + // Compute Qdp from updated Q + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV*qsize), [&] (const int k) { + const int ilev = k/qsize; + const int q = k%qsize; - Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); - // For BFB on restarts, Q needs to be updated after we compute Qdp - Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); - }); + Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); + // For BFB on restarts, Q needs to be updated after we compute Qdp + Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); + }); team.team_barrier(); - // Recompute rstar with updated qv, and convert updated temperature back to potential temperature - elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); - }); + // Convert updated temperature back to psuedo density virtual potential temperature + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&] (const int k) { + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); }); + + // Release WS views + ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); }); if (iop_nudge_tq or iop_nudge_uv) { @@ -494,8 +470,8 @@ apply_iop_forcing(const Real dt) // and observed quantities of T, Q, u, and if (iop_nudge_tq) { - // Compute rstar, exner and temperature from Hommexx - compute_homme_states(); + // Compute temperature + compute_temperature(); Kokkos::fence(); } @@ -571,25 +547,36 @@ apply_iop_forcing(const Real dt) // Apply relaxation const auto rtau = std::max(dt, iop_nudge_tscale); Kokkos::parallel_for("apply_domain_relaxation", - policy_homme, + policy_iop, KOKKOS_LAMBDA (const KT::MemberType& team) { - KV kv(team); - const int ie = team.league_rank(); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { - const int igp = idx/NGP; - const int jgp = idx%NGP; - - auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); - auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); - auto rstar_i = ekat::subview(rstar, ie, igp, jgp); - auto exner_i = ekat::subview(exner, ie, igp, jgp); - auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); - auto temperature_i = ekat::subview(temperature, ie, igp, jgp); - auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); - auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); - - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&](const int& k) { + + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; + + // Get temp views from workspace + auto ws = iop_wsm.get_workspace(team); + uview_1d pmid; + ws.take_many_contiguous_unsafe<1>({"pmid"},{&pmid}); + + auto ps_i = ps_dyn(ie, igp, jgp); + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels), [&](const int& k) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + }); + team.team_barrier(); + + if (iop_nudge_tq or iop_nudge_uv) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, NLEV), [&](const int& k) { if (iop_nudge_tq) { // Restrict nudging of T and qv to certain levels if requested by user // IOP pressure variable is in unitis of [Pa], while iop_nudge_tq_low/high @@ -599,22 +586,26 @@ apply_iop_forcing(const Real dt) for (int lev=k*Pack::n, p = 0; p < Pack::n && lev < max_size; ++lev, ++p) { const auto pressure_from_iop = hyam(lev)*ps0 + hybm(lev)*ps_iop; nudge_level.set(p, pressure_from_iop <= iop_nudge_tq_low*100 - and - pressure_from_iop >= iop_nudge_tq_high*100); + and + pressure_from_iop >= iop_nudge_tq_high*100); } qv_i(k).update(nudge_level, qv_mean(k) - qv_iop(k), -dt/rtau, 1.0); temperature_i(k).update(nudge_level, t_mean(k) - t_iop(k), -dt/rtau, 1.0); - // Convert updated temperature back to potential temperature - vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + // Convert updated temperature back to virtual potential temperature + const auto th = PF::calculate_theta_from_T(temperature_i(k),pmid(k)); + vtheta_dp_i(k) = PF::calculate_virtual_temperature(th,qv_i(k))*dp3d_i(k); } if (iop_nudge_uv) { u_i(k).update(u_mean(k) - u_iop(k), -dt/rtau, 1.0); v_i(k).update(v_mean(k) - v_iop(k), -dt/rtau, 1.0); } }); - }); + } + + // Release WS views + ws.release_many_contiguous<1>({&pmid}); }); } } diff --git a/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 index cf8d955fe902..3460feab2fe2 100644 --- a/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/dyn_grid_mod.F90 @@ -51,6 +51,7 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) use edge_mod_base, only: edgeVpack_nlyr, edgeVunpack_nlyr use kinds, only: real_kind, int_kind use dof_mod, only: genLocalDof + use control_mod, only: geometry ! ! Inputs ! @@ -62,6 +63,7 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) real(kind=real_kind), allocatable :: el_cg_gids (:,:,:) ! Homme's bex stuff only works with reals integer(kind=int_kind), allocatable :: el_dg_gids (:,:,:) ! Homme's getLocalDof might not work with c_int integer :: idof, ip,jp, ie, icol + logical :: is_sphere ! Get the gids allocate(el_cg_gids(np,np,nelemd)) @@ -78,6 +80,7 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) call edgeVpack_nlyr(edge,elem(ie)%desc,el_cg_gids(:,:,ie),1,0,1) enddo call bndry_exchangeV(par,edge) + is_sphere = trim(geometry) /= 'plane' do ie=1,nelemd call edgeVunpack_nlyr(edge,elem(ie)%desc,el_cg_gids(:,:,ie),1,0,1) elgids(ie) = elem(ie)%GlobalId @@ -86,8 +89,12 @@ subroutine get_my_dyn_data (dg_gids, cg_gids, elgpgp, elgids, lat, lon) idof = (ie-1)*16+(jp-1)*4+ip cg_gids(idof) = INT(el_cg_gids(ip,jp,ie),kind=c_int) dg_gids(idof) = INT(el_dg_gids(ip,jp,ie),kind=c_int) - lat(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lat * 180.0_c_double/pi - lon(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lon * 180.0_c_double/pi + lat(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lat + lon(ip,jp,ie) = elem(ie)%spherep(ip,jp)%lon + if (is_sphere) then + lat(ip,jp,ie) = lat(ip,jp,ie) * 180.0_c_double/pi + lon(ip,jp,ie) = lon(ip,jp,ie) * 180.0_c_double/pi + end if elgpgp(1,idof) = ie-1 elgpgp(2,idof) = jp-1 elgpgp(3,idof) = ip-1 diff --git a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 index 4d928c800b03..901cd18796ee 100644 --- a/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 +++ b/components/eamxx/src/dynamics/homme/interface/phys_grid_mod.F90 @@ -325,6 +325,7 @@ end function get_num_global_columns subroutine get_my_phys_data (gids, lat, lon, area, pg_type) use homme_context_mod, only: iam use shr_const_mod, only: pi=>SHR_CONST_PI + use control_mod, only: geometry ! ! Input(s) ! @@ -336,6 +337,7 @@ subroutine get_my_phys_data (gids, lat, lon, area, pg_type) ! integer :: pgN, load_bal, idof, ndofs type(pg_specs_t), pointer :: pgs + logical :: is_sphere ! Possible values for pg_type: ! 0 : physics grid on GLL nodes @@ -364,10 +366,15 @@ subroutine get_my_phys_data (gids, lat, lon, area, pg_type) ! TODO: when you enable twin columns, you'll have to manually ! do the search, since you can't just grab the offset-ed entries ndofs = get_num_local_columns (pgN) + is_sphere = trim(geometry) /= 'plane' do idof=1,ndofs gids(idof) = pgs%g_dofs(pgs%g_dofs_offsets(iam+1) + idof) - lat(idof) = pgs%g_lat (pgs%g_dofs_offsets(iam+1) + idof) * 180.0_c_double / pi - lon(idof) = pgs%g_lon (pgs%g_dofs_offsets(iam+1) + idof) * 180.0_c_double / pi + lat(idof) = pgs%g_lat (pgs%g_dofs_offsets(iam+1) + idof) + lon(idof) = pgs%g_lon (pgs%g_dofs_offsets(iam+1) + idof) + if (is_sphere) then + lat(idof) = lat(idof) * 180.0_c_double / pi + lon(idof) = lon(idof) * 180.0_c_double / pi + end if area(idof) = pgs%g_area(pgs%g_dofs_offsets(iam+1) + idof) enddo @@ -542,6 +549,14 @@ subroutine phys_grid_init (pgN) use gllfvremap_mod, only: gfr_init use homme_context_mod, only: elem, par use dimensions_mod, only: nelem, nelemd +#ifdef HAVE_MOAB + use seq_comm_mct, only: MHID, MHFID ! id of homme moab coarse and fine applications + use seq_comm_mct, only: ATMID + use seq_comm_mct, only: mhpgid ! id of pgx moab application + use semoab_mod, only: create_moab_meshes + use iMOAB, only : iMOAB_RegisterApplication + use iso_c_binding +#endif ! ! Input(s) ! @@ -554,6 +569,10 @@ subroutine phys_grid_init (pgN) character(2) :: str type(pg_specs_t), pointer :: pg +#ifdef HAVE_MOAB + integer :: ATM_ID1 + character*32 appname +#endif pg => pg_specs(pgN) if (pg%inited) then @@ -605,6 +624,48 @@ subroutine phys_grid_init (pgN) call compute_global_dofs (pg) call compute_global_coords (pg) call compute_global_area (pg) +#ifdef HAVE_MOAB + if (pgN > 0) then + appname="HM_COARSE"//C_NULL_CHAR + ATM_ID1 = 120 ! + ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, MHID) + if (ierr > 0 ) & + call abortmp('Error: cannot register moab app') + if(par%masterproc) then + write(iulog,*) " " + write(iulog,*) "register MOAB app:", trim(appname), " MHID=", MHID + write(iulog,*) " " + endif + appname="HM_FINE"//C_NULL_CHAR + ATM_ID1 = 119 ! this number should not conflict with other components IDs; how do we know? + ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, MHFID) + if (ierr > 0 ) & + call abortmp('Error: cannot register moab app for fine mesh') + if(par%masterproc) then + write(iulog,*) " " + write(iulog,*) "register MOAB app:", trim(appname), " MHFID=", MHFID + write(iulog,*) " " + endif + appname="HM_PGX"//C_NULL_CHAR + ATM_ID1 = ATMID(1) ! this number should not conflict with other components IDs; how do we know? + ! + ! in this case, we reuse the main atm id, mhid will not be used for intersection anymore + ! still, need to be careful + ierr = iMOAB_RegisterApplication(appname, par%comm, ATM_ID1, mhpgid) + if (ierr > 0 ) & + call abortmp('Error: cannot register moab app for fine mesh') + if(par%masterproc) then + write(iulog,*) " " + write(iulog,*) "register MOAB app:", trim(appname), " MHPGID=", mhpgid + write(iulog,*) " " + endif +! instance distributed moab meshes from elem structures +! 1 ) spectral coarse mesh +! 2 ) GLL fine quad mesh (used mostly for visualization) +! 3 ) pgN FV type mesh, (most of the time pg2 mesh), used for coupling with other components; + call create_moab_meshes(par, elem, pgN) + endif +#endif end subroutine phys_grid_init diff --git a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 index 34bbbedcc5c4..5e1b762270b2 100644 --- a/components/eamxx/src/mct_coupling/atm_comp_mct.F90 +++ b/components/eamxx/src/mct_coupling/atm_comp_mct.F90 @@ -17,6 +17,14 @@ module atm_comp_mct ! MPI use mpi +#ifdef HAVE_MOAB + use seq_comm_mct , only: mphaid ! atm physics grid id in MOAB, on atm pes + use iso_c_binding +#ifdef MOABCOMP + use seq_comm_mct, only: seq_comm_compare_mb_mct +#endif +#endif + implicit none save private @@ -25,6 +33,9 @@ module atm_comp_mct ! Public interfaces !-------------------------------------------------------------------------- +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) + public :: atm_init_hip_mct +#endif public :: atm_init_mct public :: atm_run_mct public :: atm_final_mct @@ -42,10 +53,31 @@ module atm_comp_mct integer :: atm_log_unit + +#ifdef HAVE_MOAB + ! to store all fields to be set in moab + integer , private :: mblsize, totalmbls, nsend, totalmbls_r, nrecv + real(r8) , allocatable, target, private :: a2x_am(:,:) ! atm to coupler, on atm mesh, on atm component pes + real(r8) , allocatable, target, private :: x2a_am(:,:) ! coupler to atm, on atm mesh, on atm component pes +#ifdef MOABCOMP + integer :: mpicom_atm_moab ! used just for mpi-reducing the difference between moab tags and mct avs + integer :: rank2 +#endif +#endif !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) + !=============================================================================== + subroutine atm_init_hip_mct() + use scream_f2c_mod, only: scream_init_hip_atm + + call scream_init_hip_atm() + + end subroutine atm_init_hip_mct +#endif + !=============================================================================== subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) use iso_c_binding, only: c_ptr, c_loc, c_int, c_char, c_bool @@ -67,7 +99,14 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) use seq_comm_mct, only: seq_comm_inst, seq_comm_name, seq_comm_suffix use shr_file_mod, only: shr_file_getunit, shr_file_setIO use shr_sys_mod, only: shr_sys_abort - +#ifdef HAVE_MOAB + use shr_kind_mod , only: cxx=>shr_kind_cxx + use mct_mod, only: mct_avect_nRattr + use iMOAB, only: iMOAB_SetDoubleTagStorage +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif +#endif ! !INPUT/OUTPUT PARAMETERS: type(ESMF_Clock) , intent(inout) :: EClock type(seq_cdata) , intent(inout) :: cdata @@ -87,14 +126,19 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) type(c_ptr) :: x2a_ptr, a2x_ptr character(len=256) :: atm_log_fname ! name of ATM log file character(CL) :: calendar ! calendar string - +#ifdef HAVE_MOAB + integer :: ent_type + character(CXX) :: tagname ! will store all seq_flds_a2x_fields +#ifdef MOABDEBUG + character*100 outfile, wopts +#endif +#endif ! TODO: read this from the namelist? character(len=256) :: yaml_fname = "./data/scream_input.yaml" character(kind=c_char,len=256), target :: yaml_fname_c, atm_log_fname_c character(len=256) :: caseid, username, hostname character(kind=c_char,len=256), target :: caseid_c, username_c, hostname_c, calendar_c logical (kind=c_bool) :: restarted_run - !------------------------------------------------------------------------------- ! Grab some data from the cdata structure (coming from the coupler) @@ -165,6 +209,14 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! Init MCT domain structure call atm_domain_mct (lsize, gsMap_atm, dom_atm) +#ifdef HAVE_MOAB + call moab_atm_phys_scream() +#ifdef MOABCOMP + mpicom_atm_moab = mpicom_atm ! just to store it to be used later in MOABCOMP + rank2 = my_task ! again, just to use it for MOABCOMP output +#endif +#endif + ! Init import/export mct attribute vectors call mct_aVect_init(x2a, rList=seq_flds_x2a_fields, lsize=lsize) call mct_aVect_init(a2x, rList=seq_flds_a2x_fields, lsize=lsize) @@ -182,12 +234,38 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) ! Init surface coupling stuff in the AD call scream_set_cpl_indices (x2a, a2x) +#ifdef HAVE_MOAB +! for MOAB driver, need to setup surface coupling for MOAB arrays that will hold +! fields data for import and export +! setup is similar to setup for the attribute vectors real data arrays (x2a%rAttr +! and a2x%rAttr) +! the major difference is that moab arrays are transposed compared to AVs arrays + mblsize = lsize + nsend = mct_avect_nRattr(a2x) + totalmbls = mblsize * nsend ! size of the double array + allocate (a2x_am(mblsize, nsend) ) + a2x_am = 0. ! initialize everything with 0 + + nrecv = mct_avect_nRattr(x2a) + totalmbls_r = mblsize * nrecv ! size of the double array used to receive + allocate (x2a_am(mblsize, nrecv) ) ! these will be received by moab tags, then used to set cam in surf data + x2a_am = 0. ! initialize everything with 0 +#endif + call scream_setup_surface_coupling (c_loc(import_field_names), c_loc(import_cpl_indices), & - c_loc(x2a%rAttr), c_loc(import_vector_components), & + c_loc(x2a%rAttr), & +#ifdef HAVE_MOAB + c_loc(x2a_am), & +#endif + c_loc(import_vector_components), & c_loc(import_constant_multiple), c_loc(do_import_during_init), & num_cpl_imports, num_scream_imports, import_field_size, & c_loc(export_field_names), c_loc(export_cpl_indices), & - c_loc(a2x%rAttr), c_loc(export_vector_components), & + c_loc(a2x%rAttr), & +#ifdef HAVE_MOAB + c_loc(a2x_am), & +#endif + c_loc(export_vector_components), & c_loc(export_constant_multiple), c_loc(do_export_during_init), & num_cpl_exports, num_scream_exports, export_field_size) @@ -195,6 +273,24 @@ subroutine atm_init_mct( EClock, cdata, x2a, a2x, NLFilename ) call string_f2c(trim(username),username_c) call string_f2c(trim(hostname),hostname_c) call scream_init_atm (caseid_c,hostname_c,username_c) +#ifdef HAVE_MOAB + ! data should be set now inside moab from import and export fields + ! do we import and export or just export at init stage ? + tagname=trim(seq_flds_a2x_fields)//C_NULL_CHAR + ent_type = 0 ! vertices, point cloud + ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, totalmbls , ent_type, a2x_am ) + if (ierr /= 0) then + print *,'moab error in setting data in a2x_am ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + end if +#ifdef MOABDEBUG + outfile = 'AtmPhys_init.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mphaid, outfile, wopts) + if (ierr > 0 ) & + call mpi_abort(mpicom_atm,ierr,mpi_ierr) +#endif +#endif end subroutine atm_init_mct @@ -202,6 +298,27 @@ end subroutine atm_init_mct subroutine atm_run_mct(EClock, cdata, x2a, a2x) use iso_c_binding, only: c_double use scream_f2c_mod, only: scream_run +#ifdef HAVE_MOAB + use seq_flds_mod, only: seq_flds_a2x_fields, seq_flds_x2a_fields + use shr_kind_mod , only: cxx=>shr_kind_cxx + use iMOAB, only: iMOAB_SetDoubleTagStorage, iMOAB_GetDoubleTagStorage +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif +#ifdef MOABCOMP + use mct_mod + use seq_comm_mct, only : num_moab_exports +#endif + + integer :: ent_type +#ifdef MOABCOMP + real(r8) :: difference + type(mct_list) :: temp_list + integer :: size_list, index_list + type(mct_string) :: mctOStr ! + character(CXX) :: mct_field, modelStr +#endif +#endif ! !INPUT/OUTPUT PARAMETERS: @@ -216,7 +333,14 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) type(mct_gGrid) , pointer :: ggrid real(R8) :: nextsw_cday ! calendar of next atm sw integer :: dt_scream - +#ifdef HAVE_MOAB + integer :: ierr + character(CXX) :: tagname ! will store all seq_flds_a2x_fields , seq_flds_x2a_fields +#ifdef MOABDEBUG + integer :: cur_atm_stepno + character*100 :: outfile, wopts, lnum +#endif +#endif !------------------------------------------------------------------------------- call seq_cdata_setptrs(cdata, & @@ -226,10 +350,47 @@ subroutine atm_run_mct(EClock, cdata, x2a, a2x) ! Get time step info call seq_timemgr_EClockGetData (EClock, next_cday=nextsw_cday, dtime=dt_scream) - +#ifdef HAVE_MOAB + +#ifdef MOABCOMP + ! loop over all fields in seq_flds_x2a_fields + call mct_list_init(temp_list ,seq_flds_x2a_fields) + size_list=mct_list_nitem (temp_list) + ent_type = 0 ! entity type is vertex for phys atm + if (rank2 .eq. 0) print *, num_moab_exports, trim(seq_flds_x2a_fields) + modelStr ='atm run' + do index_list = 1, size_list + call mct_list_get(mctOStr,index_list,temp_list) + mct_field = mct_string_toChar(mctOStr) + tagname= trim(mct_field)//C_NULL_CHAR + call seq_comm_compare_mb_mct(modelStr, mpicom_atm_moab, x2a, mct_field, mphaid, tagname, ent_type, difference) + enddo + call mct_list_clean(temp_list) +#endif + + + ! import data from moab data structures + tagname=trim(seq_flds_x2a_fields)//C_NULL_CHAR + ent_type = 0 ! vertices, point cloud + ierr = iMOAB_GetDoubleTagStorage ( mphaid, tagname, totalmbls_r , ent_type, x2a_am ) + +#endif ! Run scream call scream_run( dt_scream ) +#ifdef HAVE_MOAB + ! export data to moab data structures + tagname=trim(seq_flds_a2x_fields)//C_NULL_CHAR + ent_type = 0 ! vertices, point cloud + ierr = iMOAB_SetDoubleTagStorage ( mphaid, tagname, totalmbls , ent_type, a2x_am ) +#ifdef MOABDEBUG + call seq_timemgr_EClockGetData( EClock, stepno=cur_atm_stepno ) + write(lnum,"(I0.2)")cur_atm_stepno + outfile = 'AtmPhys_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mphaid, outfile, wopts) +#endif +#endif ! Set time of next radiadtion computation call seq_infodata_PutData(infodata, nextsw_cday=nextsw_cday) @@ -343,4 +504,180 @@ subroutine atm_domain_mct( lsize, gsMap_atm, dom_atm ) call mct_gGrid_importRAttr(dom_atm,"aream",data1,lsize) end subroutine atm_domain_mct +#ifdef HAVE_MOAB +! as part of initialization, moab version of the MCT type grid needs to be instanced too +! it is what we call "point cloud mesh", it contains just the vertices corresponding to the +! grid for surface coupling +! its tags are corresponding to the fields from a2x and x2a attribute vectors, and its +! GLOBAL_ID tag corresponds to the global degrees of freedom in global atmosphere + subroutine moab_atm_phys_scream() + + use iMOAB, only : iMOAB_RegisterApplication, iMOAB_CreateVertices, iMOAB_WriteMesh, & + iMOAB_DefineTagStorage, iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & + iMOAB_ResolveSharedEntities, iMOAB_UpdateMeshInfo + use iso_c_binding, only: c_int, c_loc + use scream_f2c_mod, only: scream_get_num_global_cols, scream_get_num_local_cols, & + scream_get_local_cols_gids + use scream_f2c_mod, only: scream_get_cols_latlon, scream_get_cols_area + use seq_flds_mod, only: seq_flds_dom_fields + use shr_kind_mod , only: r8 => shr_kind_r8, cl=>shr_kind_cl, cxx=>shr_kind_cxx + use shr_const_mod, only: SHR_CONST_PI + use seq_flds_mod, only: seq_flds_a2x_fields, seq_flds_x2a_fields + + ! Local variables + ! + integer(kind=c_int) :: num_local_cols, num_global_cols +#ifdef MOABDEBUG + character*100 outfile, wopts +#endif + character(CXX) :: tagname ! will store all seq_flds_a2x_fields + character*32 appname + integer :: ATM_PHYS ! used as global identifier for iMOAB app on pphys grid atmosphere (200+ atm id) + !------------------------------------------------------------------- + ! + ! Local Variables + ! + integer :: ierr,mpi_ierr ! error codes + integer :: i ! for loops along dofs + integer :: tagtype, tagindex, numco, ent_type + real(r8), pointer :: data1(:) + real(r8), pointer :: data2(:) ! temporary + integer(kind=c_int), allocatable, target :: col_gids(:) + real(r8), allocatable, target :: moab_vert_coords(:) + real(r8) :: latv, lonv + !------------------------------------------------------------------- + + appname="ATM_PHYS_SCREAM"//C_NULL_CHAR + ATM_PHYS = 200 + ATM_ID ! + ierr = iMOAB_RegisterApplication(appname, mpicom_atm, ATM_PHYS, mphaid) + if (ierr > 0 ) then + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + if (my_task == master_task) then + print *, " moab_atm_phys_scream:: register MOAB app:", trim(appname), " mphaid=", mphaid + endif + ! find global ids + num_local_cols = scream_get_num_local_cols() + num_global_cols = scream_get_num_global_cols() + + allocate(col_gids(num_local_cols)) + + call scream_get_local_cols_gids(c_loc(col_gids)) + + ! Fill in correct values for domain components + allocate(moab_vert_coords(num_local_cols*3)) + allocate( data1(num_local_cols), data2(num_local_cols) ) + call scream_get_cols_latlon(c_loc(data1),c_loc(data2)) + do i=1,num_local_cols + latv = data1(i) * SHR_CONST_PI/180 + lonv = data2(i) * SHR_CONST_PI/180 + moab_vert_coords(3*i-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*i-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*i )=SIN(latv) + enddo + ierr = iMOAB_CreateVertices(mphaid, num_local_cols*3, 3, moab_vert_coords) + if (ierr > 0 ) then + print *, "Error: fail to create MOAB vertices in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + tagtype = 0 ! dense, integer + numco = 1 + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, "Error: fail to define GLOBAL_ID tag in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ent_type = 0 ! vertex type + ierr = iMOAB_SetIntTagStorage(mphaid, tagname, num_local_cols , ent_type, col_gids) + if (ierr > 0 ) then + print *, "Error: fail to set GLOBAL_ID tag in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ierr = iMOAB_ResolveSharedEntities( mphaid, num_local_cols, col_gids) + if (ierr > 0 ) then + print *, "Error: fail to resolve shared ents in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ierr = iMOAB_UpdateMeshInfo(mphaid) + if (ierr > 0 ) then + print *, "Error: fail to update mesh info in phys atm model" + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + tagname=trim(seq_flds_dom_fields)//C_NULL_CHAR ! mask is double too lat:lon:hgt:area:aream:mask:frac + tagtype = 1 ! dense, double + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, 'Error: fail to create tags from seq_flds_dom_fields ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + call scream_get_cols_area(c_loc(data1)) + tagname = 'area'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set area ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + ! Mask and frac are both exactly 1 + data1 = 1.0 + tagname = 'mask'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set mask ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + tagname = 'frac'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set frac ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + + !call mct_gGrid_importRAttr(dom_atm,"mask",data1,lsize) + !call mct_gGrid_importRAttr(dom_atm,"frac",data1,lsize) + + ! Aream is computed by mct, so give invalid initial value + data1 = -9999.0_R8 + tagname = 'aream'//C_NULL_CHAR ! + ierr = iMOAB_SetDoubleTagStorage(mphaid, tagname, num_local_cols , ent_type, data1) + if (ierr > 0 ) then + print *, 'Error: fail to set aream ' + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + +#ifdef MOABDEBUG + outfile = 'AtmPhys.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mphaid, outfile, wopts) + if (ierr > 0 ) then + print *, "Error: fail to write PhysAtm mesh " + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif +#endif + ! define fields seq_flds_a2x_fields + tagtype = 1 ! dense, double + numco = 1 ! one value per vertex / entity + tagname = trim(seq_flds_a2x_fields)//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, "Error: fail to define seq_flds_a2x_fields " + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + ! make sure this is defined too; it could have the same fields, but in different order, or really different + ! fields; need to make sure we have them + tagname = trim(seq_flds_x2a_fields)//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mphaid, tagname, tagtype, numco, tagindex ) + if (ierr > 0 ) then + print *, "Error: fail to define seq_flds_x2a_fields " + call mpi_abort(mpicom_atm,ierr,mpi_ierr) + endif + deallocate(col_gids) + deallocate(data1) + deallocate(data2) + end subroutine moab_atm_phys_scream +#endif + end module atm_comp_mct diff --git a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 index df96edab0f06..dc5be4de373f 100644 --- a/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 +++ b/components/eamxx/src/mct_coupling/scream_cpl_indices.F90 @@ -6,7 +6,7 @@ module scream_cpl_indices private ! Focus only on the ones that scream imports/exports (subsets of x2a and a2x) - integer, parameter, public :: num_scream_imports = 16 + integer, parameter, public :: num_scream_imports = 19 integer, parameter, public :: num_scream_exports = 17 integer, public :: num_cpl_imports, num_cpl_exports, import_field_size, export_field_size @@ -89,6 +89,9 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_field_names(14) = 'surf_evap' import_field_names(15) = 'ocnfrac' import_field_names(16) = 'landfrac' + import_field_names(17) = 'icefrac' + import_field_names(18) = 'fv' + import_field_names(19) = 'ram1' ! CPL indices import_cpl_indices(1) = mct_avect_indexra(x2a,'Sx_avsdr') @@ -106,7 +109,10 @@ subroutine scream_set_cpl_indices (x2a, a2x) import_cpl_indices(13) = mct_avect_indexra(x2a,'Faxx_sen') import_cpl_indices(14) = mct_avect_indexra(x2a,'Faxx_evap') import_cpl_indices(15) = mct_avect_indexra(x2a,'Sf_ofrac') - import_cpl_indices(16) = mct_avect_indexra(x2a,'Sf_lfrac') + import_cpl_indices(16) = mct_avect_indexra(x2a,'Sf_lfrac') + import_cpl_indices(17) = mct_avect_indexra(x2a,'Sf_ifrac') + import_cpl_indices(18) = mct_avect_indexra(x2a,'Sl_fv') + import_cpl_indices(19) = mct_avect_indexra(x2a,'Sl_ram1') ! Vector components import_vector_components(11) = 0 diff --git a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp index 0bdf90eeb71c..8bf027e10bfb 100644 --- a/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp +++ b/components/eamxx/src/mct_coupling/scream_cxx_f90_interface.cpp @@ -21,6 +21,10 @@ #include "ekat/ekat_pack.hpp" #include "ekat/ekat_assert.hpp" +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) +#include +#endif + // Anonymous namespace, for some utility functions namespace { @@ -53,10 +57,23 @@ void fpe_guard_wrapper (const Lambda& f) { try { f(); } catch (std::exception &e) { + // Print exception msg, then call MPI_Abort fprintf(stderr, "%s\n", e.what()); + + // Get raw comm before cleaning up singleton auto& c = ScreamContext::singleton(); + auto raw_comm = c.get().mpi_comm(); c.clean_up(); - throw; + + MPI_Abort (raw_comm,1); + } catch (...) { + + // Get raw comm before cleaning up singleton + auto& c = ScreamContext::singleton(); + auto raw_comm = c.get().mpi_comm(); + c.clean_up(); + + MPI_Abort (raw_comm,1); } // Restore the FPE flag as it was when control was handed to us. @@ -161,11 +178,19 @@ void scream_create_atm_instance (const MPI_Fint f_comm, const int atm_id, } void scream_setup_surface_coupling (const char*& import_field_names, int*& import_cpl_indices, - double*& x2a_ptr, int*& import_vector_components, + double*& x2a_ptr, +#ifdef HAVE_MOAB + double*& x2a_moab_ptr, +#endif + int*& import_vector_components, double*& import_constant_multiple, bool*& do_import_during_init, const int& num_cpl_imports, const int& num_scream_imports, const int& import_field_size, char*& export_field_names, int*& export_cpl_indices, - double*& a2x_ptr, int*& export_vector_components, + double*& a2x_ptr, +#ifdef HAVE_MOAB + double*& a2x_moab_ptr, +#endif + int*& export_vector_components, double*& export_constant_multiple, bool*& do_export_during_init, const int& num_cpl_exports, const int& num_scream_exports, const int& export_field_size) { @@ -193,15 +218,27 @@ void scream_setup_surface_coupling (const char*& import_field_names, int*& impor ad.setup_surface_coupling_data_manager(scream::SurfaceCouplingTransferType::Import, num_cpl_imports, num_scream_imports, import_field_size, x2a_ptr, +#ifdef HAVE_MOAB + x2a_moab_ptr, +#endif names_in[0], import_cpl_indices, import_vector_components, import_constant_multiple, do_import_during_init); ad.setup_surface_coupling_data_manager(scream::SurfaceCouplingTransferType::Export, num_cpl_exports, num_scream_exports, export_field_size, a2x_ptr, +#ifdef HAVE_MOAB + a2x_moab_ptr, +#endif names_out[0], export_cpl_indices, export_vector_components, export_constant_multiple, do_export_during_init); }); } +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) +void scream_init_hip_atm () { + hipInit(0); +} +#endif + void scream_init_atm (const char* caseid, const char* hostname, const char* username) diff --git a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 index 51c3eca91230..74b45f8330b9 100644 --- a/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 +++ b/components/eamxx/src/mct_coupling/scream_f2c_mod.F90 @@ -47,11 +47,19 @@ end subroutine scream_get_cols_area ! responsible to handle import/export operation from/into the component ! coupler surface fluxes/state structures subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices, & - x2a_ptr, import_vector_components, & + x2a_ptr, & +#ifdef HAVE_MOAB + x2a_moab_ptr, & +#endif + import_vector_components, & import_constant_multiple, do_import_during_init, & num_cpl_imports, num_scream_imports, import_field_size, & export_field_names, export_cpl_indices, & - a2x_ptr, export_vector_components, & + a2x_ptr, & +#ifdef HAVE_MOAB + a2x_moab_ptr, & +#endif + export_vector_components, & export_constant_multiple, do_export_during_init, & num_cpl_exports, num_scream_exports, export_field_size) bind(c) use iso_c_binding, only: c_ptr, c_int @@ -64,11 +72,19 @@ subroutine scream_setup_surface_coupling (import_field_names, import_cpl_indices type(c_ptr), intent(in) :: export_field_names, export_cpl_indices, & a2x_ptr, export_vector_components, & export_constant_multiple, do_export_during_init +#ifdef HAVE_MOAB + type(c_ptr), intent(in) :: x2a_moab_ptr, a2x_moab_ptr +#endif integer(kind=c_int), intent(in) :: num_cpl_imports, num_scream_imports, & num_cpl_exports, num_scream_exports integer(kind=c_int), intent(in) :: import_field_size, export_field_size end subroutine scream_setup_surface_coupling +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) + subroutine scream_init_hip_atm () bind(c) + end subroutine scream_init_hip_atm +#endif + ! This subroutine performs completes the initialization of the atm instance. ! In particular, this routine must be called *after* scream_create_atm_instance, ! and *after* scream_setup_surface_coupling. diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index 40f43cbf3a7e..9874f79f9eb6 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -43,7 +43,11 @@ add_subdirectory(${EXTERNALS_SOURCE_DIR}/mam4xx ${CMAKE_BINARY_DIR}/externals/ma add_library(mam eamxx_mam_microphysics_process_interface.cpp eamxx_mam_optics_process_interface.cpp - eamxx_mam_aci_process_interface.cpp) + eamxx_mam_dry_deposition_process_interface.cpp + eamxx_mam_aci_process_interface.cpp + eamxx_mam_wetscav_process_interface.cpp + eamxx_mam_srf_and_online_emissions_process_interface.cpp + eamxx_mam_constituent_fluxes_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp index 03b04ac739a0..8bb52c918d23 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_functions.hpp @@ -198,23 +198,6 @@ void store_liquid_cloud_fraction( }); } -void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, - MAMAci::const_view_2d pdel, - const int nlev, - // output - MAMAci::view_2d rpdel) { - Kokkos::parallel_for( - team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { - const int icol = team.league_rank(); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, 0, nlev), [&](int kk) { - EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), - "Error: pdel should be > 0.\n"); - rpdel(icol, kk) = 1 / pdel(icol, kk); - }); - }); -} - void call_function_dropmixnuc( haero::ThreadTeamPolicy team_policy, const Real dt, mam_coupling::DryAtmosphere &dry_atmosphere, const MAMAci::view_2d rpdel, @@ -397,7 +380,7 @@ void call_function_dropmixnuc( progs_at_col, haero_atm, state_q_at_lev_col, klev); // get the start index for aerosols species in the state_q array - int istart = mam4::aero_model::pcnst - mam4::ndrop::ncnst_tot; + int istart = mam4::utils::aero_start_ind(); // create colum views of state_q for(int icnst = istart; icnst < mam4::aero_model::pcnst; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp index e919aa066b47..ef2c0f2b2882 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.cpp @@ -253,7 +253,7 @@ void MAMAci::set_grids( frz_unit, grid_name); // heterogeneous freezing by deposition nucleation [cm^-3 s^-1] - add_field("hetfrz_depostion_nucleation_tend", scalar3d_layout_mid, + add_field("hetfrz_deposition_nucleation_tend", scalar3d_layout_mid, frz_unit, grid_name); } // function set_grids ends @@ -390,8 +390,8 @@ void MAMAci::initialize_impl(const RunType run_type) { get_field_out("hetfrz_immersion_nucleation_tend").get_view(); hetfrz_contact_nucleation_tend_ = get_field_out("hetfrz_contact_nucleation_tend").get_view(); - hetfrz_depostion_nucleation_tend_ = - get_field_out("hetfrz_depostion_nucleation_tend").get_view(); + hetfrz_deposition_nucleation_tend_ = + get_field_out("hetfrz_deposition_nucleation_tend").get_view(); //--------------------------------------------------------------------------------- // Allocate memory for the class members @@ -597,7 +597,7 @@ void MAMAci::run_impl(const double dt) { // output cloud_frac_, cloud_frac_prev_); - compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, + mam_coupling::compute_recipical_pseudo_density(team_policy, dry_atm_.p_del, nlev_, // output rpdel_); @@ -642,7 +642,7 @@ void MAMAci::run_impl(const double dt) { team_policy, hetfrz_, dry_atm_, dry_aero_, factnum_, dt, nlev_, // ## output to be used by the other processes ## hetfrz_immersion_nucleation_tend_, hetfrz_contact_nucleation_tend_, - hetfrz_depostion_nucleation_tend_, + hetfrz_deposition_nucleation_tend_, // work arrays diagnostic_scratch_); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp index fc930ea11d7b..63f87dd9f246 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_aci_process_interface.hpp @@ -123,7 +123,7 @@ class MAMAci final : public scream::AtmosphereProcess { // added correctly to the cloud-micorphysics scheme. view_2d hetfrz_immersion_nucleation_tend_; view_2d hetfrz_contact_nucleation_tend_; - view_2d hetfrz_depostion_nucleation_tend_; + view_2d hetfrz_deposition_nucleation_tend_; view_2d diagnostic_scratch_[hetro_scratch_]; diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp new file mode 100644 index 000000000000..602fc9ebf894 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_functions.hpp @@ -0,0 +1,84 @@ +#ifndef EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP +#define EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP + +#include +#include + +namespace scream { + +namespace { + +void update_gas_aerosols_using_constituents( + const int ncol, const int nlev, const double dt, + const mam_coupling::DryAtmosphere &dry_atm, + const MAMConstituentFluxes::const_view_2d &constituent_fluxes, + // output + const mam_coupling::AerosolState &wet_aero) { + using C = physics::Constants; + static constexpr auto gravit = C::gravit; // Gravity [m/s2] + static constexpr int pcnst = mam4::aero_model::pcnst; + + // Declare local variables + const int surface_lev = nlev - 1; + + // get the start index for gas species in the state_q array + int istart = mam4::utils::gasses_start_ind(); + + // number of constituents to update (currently updating only MAM4xx + // constituents) + const int nconstituents = pcnst - istart; + + // Create a policy to loop over columns annd number of constituents + // to update + // FIXME: TODO:We don't need a team for "nconstituents", so we can make the + // kookos_for simple by using just ncols + const auto policy = ekat::ExeSpaceUtils:: + get_default_team_policy(ncol, nconstituents); + + // Loop through all columns to update tracer mixing rations + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + + //---------------------------------------------------------------------- + // To form EAM like state%q array, we need prognostics (gas and aerosol + // mmrs) atmosphere (qv, qc, nc, ni, etc.) + //---------------------------------------------------------------------- + + // Get prognostics + mam4::Prognostics progs_at_col = + mam_coupling::aerosols_for_column(wet_aero, // output + icol); // input + // Get atmospheric quantities + const haero::Atmosphere haero_atm = + atmosphere_for_column(dry_atm, // output + icol); // input + + // Form state%q like array at surface level + Real state_q_at_surf_lev[pcnst] = {}; + mam4::utils::extract_stateq_from_prognostics( + progs_at_col, haero_atm, // input + state_q_at_surf_lev, // output + surface_lev); // input + + // Compute the units conversion factor (kg/m2/s to kg/kg) + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del(icol, surface_lev) != 0, + "Error! dry_atm.pdel must be non-zero!\n"); + const Real rpdel = 1.0 / dry_atm.p_del(icol, surface_lev); + const Real unit_factor = dt * gravit * rpdel; + + // Update state vector with constituent fluxes + for(int icnst = istart; icnst < pcnst; ++icnst) { + state_q_at_surf_lev[icnst] += + constituent_fluxes(icol, icnst) * unit_factor; + } + mam4::utils::inject_stateq_to_prognostics(state_q_at_surf_lev, // input + progs_at_col, // output + surface_lev); // input + }); // icol loop +} + +} // namespace +} // namespace scream + +#endif // ifdef EAMXX_MAM_CONSTITUTE_FLUXES_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp new file mode 100644 index 000000000000..440e6e3dbdca --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.cpp @@ -0,0 +1,311 @@ +#include +#include +namespace scream { + +// ================================================================ +// Constructor +// ================================================================ +MAMConstituentFluxes::MAMConstituentFluxes(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ================================================================ +// SET_GRIDS +// ================================================================ +void MAMConstituentFluxes::set_grids( + const std::shared_ptr grids_manager) { + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + using namespace ekat::units; + auto q_unit = kg / kg; // units of mass mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + auto nondim = ekat::units::Units::nondimensional(); + + FieldLayout scalar2d = grid_->get_2d_scalar_layout(); + FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); + + static constexpr int pcnst = mam4::aero_model::pcnst; + + const FieldLayout scalar2d_pcnct = + grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- + // ----------- Atmospheric quantities ------------- + // Specific humidity [kg/kg](Require only for building DS) + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid number mixing ratio [1/kg](Require only for building DS) + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Cloud ice number mixing ratio [1/kg](Require only for building DS) + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints (Require only for building + // DS) + add_field("omega", scalar3d_mid, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_mid, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + + // Planetary boundary layer height [m] (Require only for building DS) + add_field("pbl_height", scalar2d, m, grid_name); + + // cloud fraction [nondimensional] computed by eamxx_cld_fraction_process + add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); + + static constexpr Units m2(m * m, "m2"); + static constexpr Units s2(s * s, "s2"); + + // Surface geopotential [m2/s2] (Require only for building DS) + add_field("phis", scalar2d, m2 / s2, grid_name); + + // Constituent fluxes at the surface (gasses and aerosols) + //[units: kg/m2/s (mass) or #/m2/s (number)] + add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, + grid_name); + + // --------------------------------------------------------------------- + // These variables are "Updated" or inputs/outputs for the process + // --------------------------------------------------------------------- + // NOTE: Cloud borne aerosols are not updated in this process but are included + // to create data structures. + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int mode = 0; mode < mam_coupling::num_aero_modes(); ++mode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const std::string int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(mode); + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, + "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const std::string cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(mode); + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const std::string int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(mode, a); + if(not int_mmr_field_name.empty()) { + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // NOTE: DO NOT add cld borne aerosols to the "tracer" group as these are + // NOT advected + const std::string cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(mode, a); + if(not cld_mmr_field_name.empty()) { + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); + } + } // end for loop num species + } // end for loop for num modes + + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const std::string gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } // end for loop num gases + +} // set_grid + +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMConstituentFluxes::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ================================================================ +// INIT_BUFFERS +// ================================================================ +// ON HOST, initializes the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMConstituentFluxes::init_buffers( + const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG( + used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMConstituentFluxes."); +} + +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ +void MAMConstituentFluxes::initialize_impl(const RunType run_type) { + // --------------------------------------------------------------- + // Input fields read in from IC file, namelist or other processes + // --------------------------------------------------------------- + + // Populate the wet atmosphere state with views from fields + // FIMXE: specifically look which among these are actually used by the process + wet_atm_.qv = get_field_in("qv").get_view(); + + // Following wet_atm vars are required only for building DS + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + // Populate the dry atmosphere state with views from fields + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + + // Following dry_atm vars are required only for building DS + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now + + // Constituent fluxes at the surface (gasses and aerosols) [kg/m2/s] + constituent_fluxes_ = + get_field_in("constituent_fluxes").get_view(); + + // interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const std::string int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const std::string cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const std::string int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(not int_mmr_field_name.empty()) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const std::string cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(not cld_mmr_field_name.empty()) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const std::string gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); + } + +} // end initialize_impl() + +// ================================================================ +// RUN_IMPL +// ================================================================ +void MAMConstituentFluxes::run_impl(const double dt) { + // ------------------------------------------------------------------- + // (LONG) NOTE: The following code is an adaptation of cflx.F90 code in + // E3SM. In EAMxx, all constituents are considered "wet" (or have wet + // mixing ratios), we are *not* doing any wet to dry conversions in the + // for this process. We are simply updating the MAM4xx tracers using the + // "constituent fluxes". + // We are converting wet atm to dry atm. Since we do not use or update + // any of the water constituents (qc, qv, qi etc.), we should be okay + // to do this conversion. We need to do this conversion as our function + // are built following HAERO data structures. + // ------------------------------------------------------------------- + + // Compute vertical layer heights and updraft velocity. We need these to fully + // populate dry_atm_, so that we can form a HAERO atmosphere object. HAERO + // atmosphere object is used to for state%q like array. + // NOTE: We cannot pass a member of the interface class (in this case, MAMConstituentFluxes) + // inside a parallel_for. Instead, we must create a soft copy of each member. + const auto & wet_atm = wet_atm_; + const auto & dry_atm= dry_atm_; + + auto lambda = + KOKKOS_LAMBDA(const Kokkos::TeamPolicy::member_type &team) { + const int icol = team.league_rank(); // column index + compute_dry_mixing_ratios(team, wet_atm, // in + dry_atm, // out + icol); // in + team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, // in + dry_atm, // out + icol); // in + compute_updraft_velocities(team, wet_atm, // in + dry_atm, // out + icol); // in + }; + // policy + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + Kokkos::parallel_for("mam_cfi_compute_updraft", scan_policy, lambda); + + update_gas_aerosols_using_constituents(ncol_, nlev_, dt, dry_atm_, + constituent_fluxes_, + // output + wet_aero_); +} // run_impl ends + +// ============================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp new file mode 100644 index 000000000000..7d54761a2975 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_constituent_fluxes_interface.hpp @@ -0,0 +1,76 @@ +#ifndef EAMXX_MAM_CONSTITUENT_FLUXES_HPP +#define EAMXX_MAM_CONSTITUENT_FLUXES_HPP + +// For declaring contituent fluxes class derived from atm process +// class +#include + +// For MAM4 aerosol configuration +#include +#include + +namespace scream { + +// The process responsible for applying MAM4 constituent fluxes. The +// AD stores exactly ONE instance of this class in its list of subcomponents. +class MAMConstituentFluxes final : public scream::AtmosphereProcess { + public: + using KT = ekat::KokkosTypes; + using const_view_2d = Field::view_dev_t; + + private: + // number of horizontal columns + int ncol_, nlev_; + + // Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + // aerosol state variables + mam_coupling::AerosolState wet_aero_; + + // buffer for sotring temporary variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + const_view_2d constituent_fluxes_; + + public: + // Constructor + MAMConstituentFluxes(const ekat::Comm &comm, + const ekat::ParameterList ¶ms); + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_constituent_fluxes"; } + + // grid + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + +}; // MAMConstituentFluxes + +} // namespace scream + +#endif // EAMXX_MAM_CONSTITUENT_FLUXES_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp new file mode 100644 index 000000000000..f5423e8eb1cf --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_functions.hpp @@ -0,0 +1,205 @@ +#ifndef EAMXX_MAM_DRY_DEPOSITION_FUNCTIONS_HPP +#define EAMXX_MAM_DRY_DEPOSITION_FUNCTIONS_HPP + +#include +#include +#include +#include + +namespace scream { + +namespace { +void compute_tendencies( + // inputs + const int ncol, const int nlev, const double dt, + const MAMDryDep::const_view_1d obklen, + const MAMDryDep::const_view_1d surfric, + const MAMDryDep::const_view_1d landfrac, + const MAMDryDep::const_view_1d icefrac, + const MAMDryDep::const_view_1d ocnfrac, + const MAMDryDep::const_view_1d friction_velocity, + const MAMDryDep::const_view_1d aerodynamical_resistance, + MAMDryDep::view_3d qtracers, MAMDryDep::view_2d fraction_landuse_, + const MAMDryDep::const_view_3d dgncur_awet_, + const MAMDryDep::const_view_3d wet_dens_, + const mam_coupling::DryAtmosphere dry_atm, + const mam_coupling::AerosolState dry_aero, + + // input-outputs + MAMDryDep::view_3d qqcw_, + + // outputs + MAMDryDep::view_3d ptend_q, MAMDryDep::view_2d aerdepdrycw, + MAMDryDep::view_2d aerdepdryis, + + // work arrays + MAMDryDep::view_2d rho_, MAMDryDep::view_4d vlc_dry_, + MAMDryDep::view_3d vlc_trb_, MAMDryDep::view_4d vlc_grv_, + MAMDryDep::view_3d dqdt_tmp_) { + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + ncol, nlev); + + // Parallel loop over all the columns + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { + static constexpr int num_aero_species = + mam_coupling::num_aero_species(); + + const int icol = team.league_rank(); + // Parallel loop over all the levels to populate qtracers array using + // dry_aero + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, nlev), [&](const int lev) { + for(int mode = 0; mode < num_aero_modes; ++mode) { + int icnst = mam4::ConvProc::numptrcw_amode(mode); + qtracers(icol, lev, icnst) = + dry_aero.int_aero_nmr[mode](icol, lev); + for(int species = 0; species < num_aero_species; ++species) { + icnst = mam4::ConvProc::lmassptrcw_amode(species, mode); + if(-1 < icnst) { + qtracers(icol, lev, icnst) = + dry_aero.int_aero_mmr[mode][species](icol, lev); + } + } + } + }); // parallel_for for nlevs + team.team_barrier(); + + // Create atm and progs objects + mam4::Atmosphere atm = atmosphere_for_column(dry_atm, icol); + mam4::Prognostics progs = aerosols_for_column(dry_aero, icol); + + // Extract column data (or 1d view) from 2d views of data + mam4::ConstColumnView dgncur_awet[num_aero_modes]; + mam4::ConstColumnView wet_dens[num_aero_modes]; + + for(int i = 0; i < num_aero_modes; ++i) { + dgncur_awet[i] = ekat::subview(dgncur_awet_, icol, i); + wet_dens[i] = ekat::subview(wet_dens_, icol, i); + } + + mam4::ColumnView rho; + rho = ekat::subview(rho_, icol); + + static constexpr int n_land_type = MAMDryDep::n_land_type; + Real fraction_landuse[n_land_type]; + for(int i = 0; i < n_land_type; ++i) { + fraction_landuse[i] = fraction_landuse_(i, icol); + } + + static constexpr int nmodes = mam4::AeroConfig::num_modes(); + mam4::ColumnView vlc_dry[nmodes][MAMDryDep::aerosol_categories_]; + mam4::ColumnView vlc_grv[nmodes][MAMDryDep::aerosol_categories_]; + Real vlc_trb[nmodes][MAMDryDep::aerosol_categories_]; + + for(int i = 0; i < nmodes; ++i) { + for(int j = 0; j < MAMDryDep::aerosol_categories_; ++j) { + vlc_dry[i][j] = ekat::subview(vlc_dry_, i, j, icol); + vlc_trb[i][j] = vlc_trb_(i, j, icol); + vlc_grv[i][j] = ekat::subview(vlc_grv_, i, j, icol); + } + } + static constexpr int pcnst = mam4::aero_model::pcnst; + mam4::ColumnView qqcw[pcnst]; + mam4::ColumnView dqdt_tmp[pcnst]; + for(int i = 0; i < pcnst; ++i) { + qqcw[i] = ekat::subview(qqcw_, i, icol); + dqdt_tmp[i] = ekat::subview(dqdt_tmp_, i, icol); + } + // Extract qqcw from Prognostics + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + for(int m = 0; m < nmodes; ++m) { + qqcw[mam4::ConvProc::numptrcw_amode(m)][kk] = progs.n_mode_c[m][kk]; + for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) + if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) + qqcw[mam4::ConvProc::lmassptrcw_amode(a, m)][kk] = + progs.q_aero_c[m][a][kk]; + } + }); // parallel_for nlevs + team.team_barrier(); + bool ptend_lq[pcnst]; // currently unused + mam4::aero_model_drydep( + // inputs + team, fraction_landuse, atm.temperature, atm.pressure, + atm.interface_pressure, atm.hydrostatic_dp, + ekat::subview(qtracers, icol), dgncur_awet, wet_dens, obklen[icol], + surfric[icol], landfrac[icol], icefrac[icol], ocnfrac[icol], + friction_velocity[icol], aerodynamical_resistance[icol], dt, + // input-outputs + qqcw, + // outputs + ekat::subview(ptend_q, icol), ptend_lq, + ekat::subview(aerdepdrycw, icol), ekat::subview(aerdepdryis, icol), + // work arrays + rho, vlc_dry, vlc_trb, vlc_grv, dqdt_tmp); + }); // parallel_for for ncols +} // Compute_tendencies ends + +// Update interstitial aerosols using ptend_q tendencies +void update_interstitial_mmrs(const MAMDryDep::view_3d ptend_q, const double dt, + const int ncol, const int nlev, + // output + const mam_coupling::AerosolState dry_aero) { + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + ncol, nlev); + static constexpr int nmodes = mam4::AeroConfig::num_modes(); + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MAMDryDep::KT::MemberType &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + for(int m = 0; m < nmodes; ++m) { + dry_aero.int_aero_nmr[m](icol, kk) += + ptend_q(icol, kk, mam4::ConvProc::numptrcw_amode(m)) * dt; + for(int a = 0; a < mam4::AeroConfig::num_aerosol_ids(); ++a) + if(-1 < mam4::ConvProc::lmassptrcw_amode(a, m)) + dry_aero.int_aero_mmr[m][a](icol, kk) += + ptend_q(icol, kk, mam4::ConvProc::lmassptrcw_amode(a, m)) * + dt; + } + }); // parallel_for nlevs + }); // parallel_for icol +} // Update interstitial aerosols ends + +// Update cloud borne aerosols using qqcw +void update_cloudborne_mmrs(const MAMDryDep::view_3d qqcw, const double dt, + const int nlev_, + // output + const mam_coupling::AerosolState dry_aero) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(dry_aero.cld_aero_nmr[m], + ekat::subview(qqcw, mam4::ConvProc::numptrcw_amode(m))); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + if(dry_aero.cld_aero_mmr[m][a].data()) { + Kokkos::deep_copy( + dry_aero.cld_aero_mmr[m][a], + ekat::subview(qqcw, mam4::ConvProc::lmassptrcw_amode(a, m))); + } + } + } +} // Update cloud borne aerosols ends + +// FIXME: remove the following function after implementing file read for landuse +void populated_fraction_landuse(MAMDryDep::view_2d flu, const int ncol) { + Kokkos::parallel_for( + "populated_fraction_landuse", 1, KOKKOS_LAMBDA(int) { + static constexpr int n_land_type = MAMDryDep::n_land_type; + const Real temp[n_land_type] = { + 0.28044346587077795E-003, 0.26634987180780171E-001, + 0.16803558403621365E-001, 0.18076055155371872E-001, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.91803784897907303E+000, 0.17186036997038400E-002, + 0.00000000000000000E+000, 0.00000000000000000E+000, + 0.18448503115578840E-001}; + for(int i = 0; i < n_land_type; ++i) + for(int j = 0; j < ncol; ++j) flu(i, j) = temp[i]; + }); + Kokkos::fence(); +} + +} // namespace +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp new file mode 100644 index 000000000000..034972f85258 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.cpp @@ -0,0 +1,439 @@ +#include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" + +// Drydep functions are stored in the following hpp file +#include + +/* +----------------------------------------------------------------- +NOTES: +1. Add a CIME test and multi-process tests +2. Ensure that the submodule for MAM4xx is the main branch +3. Read file for fractional landuse +----------------------------------------------------------------- +*/ +namespace scream { + +MAMDryDep::MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ================================================================ +// SET_GRIDS +// ================================================================ +void MAMDryDep::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + + // set grid for all the inputs and outputs + // use physics grid + grid_ = grids_manager->get_grid("Physics"); + + // Name of the grid + const auto &grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + // Define the different field layouts that will be used for this process + using namespace ShortFieldTagsNames; + + // Layout for 2D (2d horiz) variable + const FieldLayout scalar2d = grid_->get_2d_scalar_layout(); + + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and + // interfaces + const FieldLayout scalar3d_mid = grid_->get_3d_scalar_layout(true); + const FieldLayout scalar3d_int = grid_->get_3d_scalar_layout(false); + + // layout for 2D (ncol, pcnst) + constexpr int pcnst = mam4::aero_model::pcnst; + const FieldLayout vector2d_pcnst = + grid_->get_2d_vector_layout(pcnst, "num_phys_constants"); + + // Layout for 4D (2d horiz X 1d vertical x number of modes) variables + // at mid points + const int num_aero_modes = mam_coupling::num_aero_modes(); + const FieldLayout vector3d_mid = grid_->get_3d_vector_layout(true, num_aero_modes, "num_modes"); + + using namespace ekat::units; + + auto q_unit = kg / kg; // units of mass mixing ratios of tracers + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + + auto nondim = ekat::units::Units::nondimensional(); + + auto m3 = m * m * m; // meter cubed + + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- + + // ----------- Atmospheric quantities ------------- + // Specific humidity [kg/kg](Require only for building DS) + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid mass mixing ratio [kg/kg](Require only for building DS) + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud ice mass mixing ratio [kg/kg](Require only for building DS) + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + + // Cloud liquid number mixing ratio [1/kg](Require only for building DS) + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Cloud ice number mixing ratio [1/kg](Require only for building DS) + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints (Require only for building + // DS) + add_field("omega", scalar3d_mid, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_mid, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + + // Planetary boundary layer height [m] (Require only for building DS) + add_field("pbl_height", scalar2d, m, grid_name); + + static constexpr auto m2 = m * m; + static constexpr auto s2 = s * s; + + // Surface geopotential [m2/s2] (Require only for building DS) + add_field("phis", scalar2d, m2 / s2, grid_name); + + //----------- Variables from microphysics scheme ------------- + + // Total cloud fraction [fraction] (Require only for building DS) + add_field("cldfrac_tot", scalar3d_mid, nondim, grid_name); + + //----------- Variables from coupler (land component)--------- + // Obukhov length [m] + add_field("obklen", scalar2d, m, grid_name); + + // Surface friction velocty or ustar[m/s] + add_field("ustar", scalar2d, m / s, grid_name); + + // Land fraction [fraction] + add_field("landfrac", scalar2d, nondim, grid_name); + + // Friction velocity from land model [m/s] + add_field("fv", scalar2d, m / s, grid_name); + + // Aerodynamical resistance from land model [s/m] + add_field("ram1", scalar2d, s / m, grid_name); + + //----------- Variables from coupler (ice component)--------- + + // Ice fraction [unitless] + add_field("icefrac", scalar2d, nondim, grid_name); + + //----------- Variables from coupler (ocean component)--------- + // Ocean fraction [unitless] + add_field("ocnfrac", scalar2d, nondim, grid_name); + + //----------- Variables from other mam4xx processes ------------ + // Geometric mean wet diameter for number distribution [m] + add_field("dgnumwet", vector3d_mid, m, grid_name); + + // Wet density of interstitial aerosol [kg/m3] + add_field("wetdens", vector3d_mid, kg / m3, grid_name); + + // --------------------------------------------------------------------- + // These variables are "updated" or inputs/outputs for the process + // --------------------------------------------------------------------- + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < num_aero_modes; ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, + "tracers"); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } + } + } + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < num_aero_modes; ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); + } + } + } + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } + + // ------------------------------------------------------------- + // These variables are "Computed" or outputs for the process + // ------------------------------------------------------------- + // FIXME: These are diagnostics, remove them from FM after initial evaluation + // surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] + add_field("deposition_flux_of_cloud_borne_aerosols", vector2d_pcnst, + 1 / m2 / s, grid_name); + // surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] + add_field("deposition_flux_of_interstitial_aerosols", + vector2d_pcnst, 1 / m2 / s, grid_name); +} // set_grids + +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ +// ON HOST, returns the number of bytes of device memory needed by +// the above. Buffer type given the number of columns and vertical +// levels +size_t MAMDryDep::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} // requested_buffer_size_in_bytes + +// ================================================================ +// INIT_BUFFERS +// ================================================================ +// ON HOST, initializeÑ• the Buffer type with sufficient memory to +// store intermediate (dry) quantities on the given number of +// columns with the given number of vertical levels. Returns the +// number of bytes allocated. +void MAMDryDep::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMDryDep."); +} // init_buffers + +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ +void MAMDryDep::initialize_impl(const RunType run_type) { + // --------------------------------------------------------------- + // Input fields read in from IC file, namelist or other processes + // --------------------------------------------------------------- + + // Populate the wet atmosphere state with views from fields + // FIMXE: specifically look which among these are actually used by the process + wet_atm_.qv = get_field_in("qv").get_view(); + + // Following wet_atm vars are required only for building DS + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + // Populate the dry atmosphere state with views from fields + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + + // Following dry_atm vars are required only for building DS + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now + + // ---- set wet/dry aerosol-related gas state data + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } + } + } + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = + get_field_out(gas_mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + //----------------------------------------------------------------- + // Allocate memory + //----------------------------------------------------------------- + const int pcnst = mam4::aero_model::pcnst; + + // Output of the the mixing ratio tendencies [kg/kg/s or 1/kg/s] + ptend_q_ = view_3d("ptend_q_", ncol_, nlev_, pcnst); + + // Deposition velocity of turbulent dry deposition [m/s] + vlc_trb_ = view_3d("vlc_trb_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_); + // Deposition velocity of gravitational settling [m/s] + vlc_grv_ = view_4d("vlc_grv_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_, nlev_); + // Deposition velocity, [m/s] + // Fraction landuse weighted sum of vlc_grv and vlc_trb + vlc_dry_ = view_4d("vlc_dry_", mam4::AeroConfig::num_modes(), + aerosol_categories_, ncol_, nlev_); + + // Work array to hold the mixing ratios [kg/kg or 1/kg] + // Packs AerosolState::int_aero_nmr and AerosolState::int_aero_nmr + // into one array. + qtracers_ = view_3d("qtracers_", ncol_, nlev_, pcnst); + + // Work array to hold the air density [kg/m3] + rho_ = view_2d("rho", ncol_, nlev_); + + // Work array to hold cloud borne aerosols mixing ratios [kg/kg or 1/kg] + // Filled with Prognostics::n_mode_c and Prognostics::q_aero_c + qqcw_ = view_3d("qqcw_", pcnst, ncol_, nlev_); + + // Work array to hold tendency for 1 species [kg/kg/s] or [1/kg/s] + dqdt_tmp_ = view_3d("dqdt_tmp_", pcnst, ncol_, nlev_); + + static constexpr int n_land_type = mam4::DryDeposition::n_land_type; + // FIXME: This should come from a file reading + // The fraction of land use for the column. [non-dimentional] + fraction_landuse_ = view_2d("fraction_landuse_", n_land_type, ncol_); + + //----------------------------------------------------------------- + // Setup preprocessing and post processing + //----------------------------------------------------------------- + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); +} // initialize_impl + +// ========================================================================================= +void MAMDryDep::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + // ------------------------------------------------------------- + // Inputs fields for the process + // ------------------------------------------------------------- + + // Geometric mean wet diameter for number distribution [m] + auto dgncur_awet_ = get_field_in("dgnumwet").get_view(); + // Wet density of interstitial aerosol [kg/m3] + auto wet_dens_ = get_field_in("wetdens").get_view(); + // Obukhov length [m] + auto obukhov_length_ = get_field_in("obklen").get_view(); + // Land fraction [unitless] + auto land_fraction_ = get_field_in("landfrac").get_view(); + // Ice fraction [unitless] + auto ice_fraction_ = get_field_in("icefrac").get_view(); + // Ocean fraction [unitless] + auto ocean_fraction_ = get_field_in("ocnfrac").get_view(); + // Friction velocity from land model [m/s] + auto friction_velocity_ = get_field_in("fv").get_view(); + // Aerodynamical resistance from land model [s/m] + auto aerodynamical_resistance_ = + get_field_in("ram1").get_view(); + // Sfc friction velocity or ustar [m/s] + auto surface_friction_velocty_ = + get_field_in("ustar").get_view(); + + // ------------------------------------------------------------- + // Output fields for the process + // ------------------------------------------------------------- + // Surface deposition flux of cloud-borne aerosols, [kg/m2/s] or [1/m2/s] + auto aerdepdrycw_ = get_field_out("deposition_flux_of_cloud_borne_aerosols") + .get_view(); + // Surface deposition flux of interstitial aerosols, [kg/m2/s] or [1/m2/s] + auto aerdepdryis_ = get_field_out("deposition_flux_of_interstitial_aerosols") + .get_view(); + + // FIXME: remove it if it read from a file + populated_fraction_landuse(fraction_landuse_, ncol_); + + // Call drydeposition and get tendencies + compute_tendencies(ncol_, nlev_, dt, obukhov_length_, + surface_friction_velocty_, land_fraction_, ice_fraction_, + ocean_fraction_, friction_velocity_, + aerodynamical_resistance_, qtracers_, fraction_landuse_, + dgncur_awet_, wet_dens_, dry_atm_, dry_aero_, + // Inouts-outputs + qqcw_, + // Outputs + ptend_q_, aerdepdrycw_, aerdepdryis_, + // work arrays + rho_, vlc_dry_, vlc_trb_, vlc_grv_, dqdt_tmp_); + Kokkos::fence(); + + // Update the interstitial aerosols using ptend. + update_interstitial_mmrs(ptend_q_, dt, ncol_, nlev_, // inputs + dry_aero_); // output + + // Update the interstitial aerosols + update_cloudborne_mmrs(qqcw_, dt, nlev_, // inputs + dry_aero_); // output + + // call post processing to convert dry mixing ratios to wet mixing ratios + // and update the state + Kokkos::parallel_for("postprocess", scan_policy, postprocess_); + Kokkos::fence(); // wait before returning to calling function +} // run_impl +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp new file mode 100644 index 000000000000..e39090cecdea --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_dry_deposition_process_interface.hpp @@ -0,0 +1,239 @@ +#ifndef EAMXX_MAM_DRYDEP_HPP +#define EAMXX_MAM_DRYDEP_HPP + +// For declaring dry deposition class derived from atm process class +#include + +// For MAM4 aerosol configuration +#include + +// For component name +#include + +namespace scream { + +// The process responsible for handling MAM4 dry deposition. The AD +// stores exactly ONE instance of this class in its list of subcomponents. +class MAMDryDep final : public scream::AtmosphereProcess { + public: + static constexpr int num_aero_modes = mam_coupling::num_aero_modes(); + static constexpr int aerosol_categories_ = + mam4::DryDeposition::aerosol_categories; + static constexpr int n_land_type = mam4::DryDeposition::n_land_type; + + using view_1d = Field::view_dev_t; + using view_2d = Field::view_dev_t; + using view_3d = Field::view_dev_t; + using view_4d = Field::view_dev_t; + using const_view_1d = Field::view_dev_t; + using const_view_3d = Field::view_dev_t; + + private: + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + // aerosol state variables + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // buffer for sotring temporary variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + + /* Note on mam4::DryDeposition::aerosol_categories = 4 + used in deposition velocity dimension defined below. These + correspond to the two attachment states and two moments: + 0 - interstitial aerosol, 0th moment (i.e., number) + 1 - interstitial aerosol, 3rd moment (i.e., volume/mass) + 2 - cloud-borne aerosol, 0th moment (i.e., number) + 3 - cloud-borne aerosol, 3rd moment (i.e., volume/mass) + see comments in the DryDeposition class in mam4xx. + */ + // Output deposition velocity of turbulent dry deposition [m/s] + // Dimensions + // [numer of modes, aerosol_categories_, num columns] + view_3d vlc_trb_; + + // Output deposition velocity of gravitational settling [m/s] + // Dimensions + // [num_modes, aerosol_categories_, num columns, num levels] + view_4d vlc_grv_; + + // Output deposition velocity, [m/s] + // fraction landuse weighted sum of vlc_grv and vlc_trb + // Dimensions + // [num_modes, aerosol_categories_, num columns, num levels] + view_4d vlc_dry_; + + + // Output of the the mixing ratio tendencies [kg/kg/s or 1/kg/s] + // Dimensions + // [num columns, num levels, mam4::aero_model::pcnst] + // Packed the same way qtracers_ is layed out. + view_3d ptend_q_; + + // Work array to hold the mixing ratios [kg/kg or 1/kg] + // Dimensions + // [num columns, num levels, mam4::aero_model::pcnst] + // Packs AerosolState::int_aero_nmr + // and AerosolState::int_aero_nmr + // into one array, hence is mixed kg/kg and 1/kg. + view_3d qtracers_; + + // Work array to hold the fraction [non-dimentional] + // of land use for column. + // Dimensions + // [MAMDryDep::n_land_type, num columns] + // Values should sum to 1. + view_2d fraction_landuse_; + + // Work array to hold the air density [kg/m3] + // Dimensions + // [num columns, num levels] + // Calculated from air pressure at layer midpoint, + // Constants::r_gas_dry_air and air temperture. + view_2d rho_; + + // Work array to hold tendency for 1 species [kg/kg/s] or [1/kg/s] + // Dimensions + // [mam4::aero_model::pcnst, num column, num level] + view_3d dqdt_tmp_; + + // Work array to hold cloud borne aerosols mixing ratios [kg/kg or 1/kg] + // Dimensions + // [mam4::aero_model::pcnst, num column, num level] + // Filled with Prognostics::n_mode_c and Prognostics::q_aero_c + view_3d qqcw_; + + public: + using KT = ekat::KokkosTypes; + + // Constructor + MAMDryDep(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // The type of subcomponent + AtmosphereProcessType type() const override { + return AtmosphereProcessType::Physics; + } + + // The name of the subcomponent + std::string name() const override { return "mam_dry_deposition"; } + + // grid + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl() override{/*Do nothing*/}; + + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + wet_aero_pre_ = wet_aero; + dry_atm_pre_ = dry_atm; + dry_aero_pre_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); + team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, dry_atm_pre_, i); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); + } // Preprocess operator() + + // local variables for preprocess struct + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + }; // Preprocess + + // Atmosphere processes often have a post-processing step prepares output + // from this process for the Field Manager. This functor implements this + // step, which is called during run_impl. + // Postprocessing functor + struct Postprocess { + Postprocess() = default; + + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_post_ = ncol; + nlev_post_ = nlev; + wet_atm_post_ = wet_atm; + wet_aero_post_ = wet_aero; + dry_atm_post_ = dry_atm; + dry_aero_post_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, + wet_aero_post_, i); + } // operator() Postprocess + + // number of horizontal columns and vertical levels + int ncol_post_, nlev_post_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_post_; + mam_coupling::DryAtmosphere dry_atm_post_; + mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; + }; // Postprocess + + private: + // pre- and postprocessing scratch pads + Preprocess preprocess_; + Postprocess postprocess_; +}; // MAMDryDep + +} // namespace scream + +#endif // EAMXX_MAM_DRYDEP_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 7ea2adfddb63..f8a34bd90d5b 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -317,6 +317,8 @@ void MAMMicrophysics::run_impl(const double dt) { // FIXME: read relevant chlorine loading data from file based on time // loop over atmosphere columns and compute aerosol microphyscs + auto some_step = step_; + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { const int icol = team.league_rank(); // column index diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp new file mode 100644 index 000000000000..850a82d0896d --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -0,0 +1,275 @@ +//#include +#include + +namespace scream { + +// ================================================================ +// Constructor +// ================================================================ +MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants. + */ +} + +// ================================================================ +// SET_GRIDS +// ================================================================ +void MAMSrfOnlineEmiss::set_grids( + const std::shared_ptr grids_manager) { + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + + ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + + using namespace ekat::units; + + static constexpr int pcnst = mam4::aero_model::pcnst; + const FieldLayout scalar2d_pcnct = + grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + + // ------------------------------------------------------------- + // These variables are "Computed" or outputs for the process + // ------------------------------------------------------------- + static constexpr Units m2(m * m, "m2"); + // Constituent fluxes of species in [kg/m2/s] + add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, + grid_name); + + // Surface emissions remapping file + auto srf_map_file = m_params.get("srf_remap_file", ""); + + // FIXME: We can extract the following info about each species + // in a separate hpp file + //-------------------------------------------------------------------- + // Init dms srf emiss data structures + //-------------------------------------------------------------------- + srf_emiss_ dms; + // File name, name and sectors + dms.data_file = m_params.get("srf_emis_specifier_for_DMS"); + dms.species_name = "dms"; + dms.sectors = {"DMS"}; + srf_emiss_species_.push_back(dms); // add to the vector + + //-------------------------------------------------------------------- + // Init so2 srf emiss data structures + //-------------------------------------------------------------------- + srf_emiss_ so2; + // File name, name and sectors + so2.data_file = m_params.get("srf_emis_specifier_for_SO2"); + so2.species_name = "so2"; + so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(so2); // add to the vector + //-------------------------------------------------------------------- + // Init bc_a4 srf emiss data structures + //-------------------------------------------------------------------- + srf_emiss_ bc_a4; + // File name, name and sectors + bc_a4.data_file = m_params.get("srf_emis_specifier_for_bc_a4"); + bc_a4.species_name = "bc_a4"; + bc_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(bc_a4); // add to the vector + + //-------------------------------------------------------------------- + // Init num_a1 srf emiss data structures + //-------------------------------------------------------------------- + srf_emiss_ num_a1; + // File name, name and sectors + num_a1.data_file = m_params.get("srf_emis_specifier_for_num_a1"); + num_a1.species_name = "num_a1"; + num_a1.sectors = {"num_a1_SO4_AGR", "num_a1_SO4_SHP", "num_a1_SO4_SLV", + "num_a1_SO4_WST"}; + srf_emiss_species_.push_back(num_a1); // add to the vector + + //-------------------------------------------------------------------- + // Init num_a2 srf emiss data structures + //-------------------------------------------------------------------- + srf_emiss_ num_a2; + // File name, name and sectors + num_a2.data_file = m_params.get("srf_emis_specifier_for_num_a2"); + num_a2.species_name = "num_a2"; + num_a2.sectors = {"num_a2_SO4_RCO", "num_a2_SO4_TRA"}; + srf_emiss_species_.push_back(num_a2); // add to the vector + + //-------------------------------------------------------------------- + // Init num_a4 srf emiss data structures + //-------------------------------------------------------------------- + srf_emiss_ num_a4; + // File name, name and sectors + num_a4.data_file = m_params.get("srf_emis_specifier_for_num_a4"); + num_a4.species_name = "num_a4"; + num_a4.sectors = { + "num_a1_BC_AGR", "num_a1_BC_ENE", "num_a1_BC_IND", "num_a1_BC_RCO", + "num_a1_BC_SHP", "num_a1_BC_SLV", "num_a1_BC_TRA", "num_a1_BC_WST", + "num_a1_POM_AGR", "num_a1_POM_ENE", "num_a1_POM_IND", "num_a1_POM_RCO", + "num_a1_POM_SHP", "num_a1_POM_SLV", "num_a1_POM_TRA", "num_a1_POM_WST"}; + srf_emiss_species_.push_back(num_a4); // add to the vector + + //-------------------------------------------------------------------- + // Init pom_a4 srf emiss data structures + //-------------------------------------------------------------------- + srf_emiss_ pom_a4; + // File name, name and sectors + pom_a4.data_file = m_params.get("srf_emis_specifier_for_pom_a4"); + pom_a4.species_name = "pom_a4"; + pom_a4.sectors = {"AGR", "ENE", "IND", "RCO", "SHP", "SLV", "TRA", "WST"}; + srf_emiss_species_.push_back(pom_a4); // add to the vector + + //-------------------------------------------------------------------- + // Init so4_a1 srf emiss data structures + //-------------------------------------------------------------------- + srf_emiss_ so4_a1; + // File name, name and sectors + so4_a1.data_file = m_params.get("srf_emis_specifier_for_so4_a1"); + so4_a1.species_name = "so4_a1"; + so4_a1.sectors = {"AGR", "SHP", "SLV", "WST"}; + srf_emiss_species_.push_back(so4_a1); + + //-------------------------------------------------------------------- + // Init so4_a2 srf emiss data structures + //-------------------------------------------------------------------- + srf_emiss_ so4_a2; + // File name, name and sectors + so4_a2.data_file = m_params.get("srf_emis_specifier_for_so4_a2"); + so4_a2.species_name = "so4_a2"; + so4_a2.sectors = {"RCO", "TRA"}; + srf_emiss_species_.push_back(so4_a2); + + //-------------------------------------------------------------------- + // Init data structures to read and interpolate + //-------------------------------------------------------------------- + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { + srfEmissFunc::init_srf_emiss_objects( + ncol_, grid_, ispec_srf.data_file, ispec_srf.sectors, srf_map_file, + // output + ispec_srf.horizInterp_, ispec_srf.data_start_, ispec_srf.data_end_, + ispec_srf.data_out_, ispec_srf.dataReader_); + } + +} // set_grid ends + +// ================================================================ +// REQUEST_BUFFER_SIZE_IN_BYTES +// ================================================================ +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +size_t MAMSrfOnlineEmiss::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); +} + +// ================================================================ +// INIT_BUFFERS +// ================================================================ +// ON HOST, initializes the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG( + used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMSrfOnlineEmiss."); +} + +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ +void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { + // --------------------------------------------------------------- + // Output fields + // --------------------------------------------------------------- + // Constituent fluxes of species in [kg/m2/s] + constituent_fluxes_ = get_field_out("constituent_fluxes").get_view(); + + // --------------------------------------------------------------- + // Allocate memory for local and work arrays + // --------------------------------------------------------------- + + // Work array to store fluxes after unit conversions to kg/m2/s + fluxes_in_mks_units_ = view_1d("fluxes_in_mks_units", ncol_); + + // Current month ( 0-based) + const int curr_month = timestamp().get_month() - 1; + + // Load the first month into data_end. + + // Note: At the first time step, the data will be moved into data_beg, + // and data_end will be reloaded from file with the new month. + + //-------------------------------------------------------------------- + // Update surface emissions from file + //-------------------------------------------------------------------- + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { + srfEmissFunc::update_srfEmiss_data_from_file( + ispec_srf.dataReader_, timestamp(), curr_month, *ispec_srf.horizInterp_, + ispec_srf.data_end_); // output + } + + //----------------------------------------------------------------- + // Setup preprocessing and post processing + //----------------------------------------------------------------- + preprocess_.initialize(constituent_fluxes_); + +} // end initialize_impl() + +// ================================================================ +// RUN_IMPL +// ================================================================ +void MAMSrfOnlineEmiss::run_impl(const double dt) { + // Zero-out output + Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); + + // Gather time and state information for interpolation + auto ts = timestamp() + dt; + + //-------------------------------------------------------------------- + // Interpolate srf emiss data + //-------------------------------------------------------------------- + + for(srf_emiss_ &ispec_srf : srf_emiss_species_) { + // Update TimeState, note the addition of dt + ispec_srf.timeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + srfEmissFunc::update_srfEmiss_timestate( + ispec_srf.dataReader_, ts, *ispec_srf.horizInterp_, + // output + ispec_srf.timeState_, ispec_srf.data_start_, ispec_srf.data_end_); + + // Call the main srfEmiss routine to get interpolated aerosol forcings. + srfEmissFunc::srfEmiss_main(ispec_srf.timeState_, ispec_srf.data_start_, + ispec_srf.data_end_, ispec_srf.data_out_); + + //-------------------------------------------------------------------- + // Modify units to MKS units (from molecules/cm2/s to kg/m2/s) + //-------------------------------------------------------------------- + // Get species index in array with pcnst dimension (e.g., state_q or + // constituent_fluxes_) + const int species_index = spcIndex_in_pcnst_.at(ispec_srf.species_name); + + // modify units from molecules/cm2/s to kg/m2/s + auto fluxes_in_mks_units = this->fluxes_in_mks_units_; + auto constituent_fluxes = this->constituent_fluxes_; + const Real mfactor = + amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; + // Parallel loop over all the columns to update units + Kokkos::parallel_for( + "fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + fluxes_in_mks_units(icol) = + ispec_srf.data_out_.emiss_sectors(0, icol) * mfactor; + constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); + }); + + } // for loop for species + Kokkos::fence(); +} // run_imple ends + +// ============================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp new file mode 100644 index 000000000000..031fb62d8b75 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -0,0 +1,133 @@ +#ifndef EAMXX_MAM_SRF_ONLINE_EMISS_HPP +#define EAMXX_MAM_SRF_ONLINE_EMISS_HPP + +#include "share/grid/remap/abstract_remapper.hpp" +#include "share/io/scorpio_input.hpp" + +// For MAM4 aerosol configuration +#include +#include + +// For declaring surface and online emission class derived from atm process +// class +#include + +// #include +#include + +namespace scream { + +// The process responsible for handling MAM4 surface and online emissions. The +// AD stores exactly ONE instance of this class in its list of subcomponents. +class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { + using KT = ekat::KokkosTypes; + using view_1d = typename KT::template view_1d; + using view_2d = typename KT::template view_2d; + + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // buffer for sotring temporary variables + mam_coupling::Buffer buffer_; + + // physics grid for column information + std::shared_ptr grid_; + + // Constituent fluxes of species in [kg/m2/s] + view_2d constituent_fluxes_; + + // Work array to store fluxes after unit conversions to kg/m2/s + view_1d fluxes_in_mks_units_; + + // Unified atomic mass unit used for unit conversion (BAD constant) + static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu + + public: + using srfEmissFunc = mam_coupling::srfEmissFunctions; + + // Constructor + MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- + + // The type of subcomponent + AtmosphereProcessType type() const { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const { return "mam_srf_online_emissions"; } + + // grid + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl(){/*Do nothing*/}; + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const view_2d &constituent_fluxes) { + constituent_fluxes_pre_ = constituent_fluxes; + } + // local variables for preprocess struct + view_2d constituent_fluxes_pre_; + }; // MAMSrfOnlineEmiss::Preprocess + + private: + // preprocessing scratch pad + Preprocess preprocess_; + + // Species index (zero-based) in tracer array with "pcnst" dimension + // FIXME: Remove the hardwired indices and use a function + // to find them from an array. + const std::map spcIndex_in_pcnst_ = { + {"so2", 12}, {"dms", 13}, {"so4_a1", 15}, + {"num_a1", 22}, {"so4_a2", 23}, {"num_a2", 27}, + {"pom_a4", 36}, {"bc_a4", 37}, {"num_a4", 39}}; + + // A struct carrying all the fields needed to read + // surface emissions of a species + struct srf_emiss_ { + // species name + std::string species_name; + + // Data file name + std::string data_file; + + // Sector names in file + std::vector sectors; + + // Data structure for reading interpolation + std::shared_ptr horizInterp_; + std::shared_ptr dataReader_; + srfEmissFunc::srfEmissTimeState timeState_; + srfEmissFunc::srfEmissInput data_start_, data_end_; + srfEmissFunc::srfEmissOutput data_out_; + }; + + // A vector for carrying emissions for all the species + std::vector srf_emiss_species_; + + // offset for converting pcnst index to gas_pcnst index + static constexpr int offset_ = + mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; + +}; // MAMSrfOnlineEmiss + +} // namespace scream + +#endif // EAMXX_MAM_SRF_ONLINE_EMISS_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp new file mode 100644 index 000000000000..2a81c1a9c427 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.cpp @@ -0,0 +1,577 @@ +#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" + +/* +----------------------------------------------------------------- +NOTES: +1. We should connect surface fluxes and add code to update the fluxes +2. Identify diagnostic variables and remove them from FM +3. Add assert statements to check output ranges +*/ + +namespace scream { + +// ========================================================================================= +MAMWetscav::MAMWetscav(const ekat::Comm &comm, + const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params) { + /* Anything that can be initialized without grid information can be + * initialized here. Like universal constants, mam wetscav options. + */ +} + +// ================================================================ +// SET_GRIDS +// ================================================================ +void MAMWetscav::set_grids( + const std::shared_ptr grids_manager) { + using namespace ekat::units; + + // The units of mixing ratio Q are technically non-dimensional. + // Nevertheless, for output reasons, we like to see 'kg/kg'. + auto q_unit = kg / kg; + auto n_unit = 1 / kg; // units of number mixing ratios of tracers + + m_grid = grids_manager->get_grid("Physics"); + const auto &grid_name = m_grid->name(); + + ncol_ = m_grid->get_num_local_dofs(); // Number of columns on this rank + nlev_ = m_grid->get_num_vertical_levels(); // Number of levels per column + const int nmodes = mam4::AeroConfig::num_modes(); // Number of modes + constexpr int pcnst = mam4::aero_model::pcnst; + + // layout for 3D (2d horiz X 1d vertical) variables at level + // midpoints/interfaces + FieldLayout scalar3d_mid = m_grid->get_3d_scalar_layout(true); + FieldLayout scalar3d_int = m_grid->get_3d_scalar_layout(false); + + // layout for 2D (1d horiz X 1d vertical) variables + FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + + // layout for 3D (ncol, nmodes, nlevs) + FieldLayout scalar3d_mid_nmodes = + m_grid->get_3d_vector_layout(true, nmodes, "nmodes"); + + // layout for 2D (ncol, pcnst) + FieldLayout scalar2d_pconst = + m_grid->get_2d_vector_layout(pcnst, "num_phys_constants"); + + // -------------------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // -------------------------------------------------------------------------- + + // ----------- Atmospheric quantities ------------- + // Specific humidity [kg/kg] + add_field("qv", scalar3d_mid, q_unit, grid_name, "tracers"); + + // cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_mid, q_unit, grid_name, "tracers"); + + // cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_mid, q_unit, grid_name, "tracers"); + + // cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_mid, n_unit, grid_name, "tracers"); + + // cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_mid, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_mid, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_mid, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_mid, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_int, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_mid, Pa, grid_name); + + // planetary boundary layer height [m] + add_field("pbl_height", scalar2d, m, grid_name); + + static constexpr auto m2 = m * m; + static constexpr auto s2 = s * s; + + // Surface geopotential [m2/s2] + add_field("phis", scalar2d, m2 / s2, grid_name); + + //----------- Variables from microphysics scheme ------------- + + // Evaporation from stratiform rain [kg/kg/s] + add_field("nevapr", scalar3d_mid, kg / kg / s, grid_name); + + // Stratiform rain production rate [kg/kg/s] + add_field("precip_total_tend", scalar3d_mid, kg / kg / s, + grid_name); + + // For variables that are non dimensional (e.g., fractions etc.) + static constexpr auto nondim = Units::nondimensional(); + + //----------- Variables from macrophysics scheme ------------- + + // Total cloud fraction [fraction] + add_field("cldfrac_liq", scalar3d_mid, nondim, grid_name); + + // --------------------------------------------------------------------- + // These variables are "updated" or inputs/outputs for the process + // --------------------------------------------------------------------- + // FIXME: we have not added code to update the surface fluxes. + // -- surface fluxes (input/outpts) for the coupler's cam_out data struture + // for the land model + + // Wet deposition of hydrophilic black carbon [kg/m2/s] + add_field("wetdep_hydrophilic_bc", scalar3d_mid, kg / m2 / s, + grid_name); + + // Dry deposition of hydrophilic black carbon [kg/m2/s] + add_field("drydep_hydrophilic_bc", scalar3d_mid, kg / m2 / s, + grid_name); + + // Wet deposition of hydrophilic organic carbon [kg/m2/s] + add_field("wetdep_hydrophilic_oc", scalar3d_mid, kg / m2 / s, + grid_name); + + // Dry deposition of hydrophilic organic carbon [kg/m2/s] + add_field("drydep_hydrophilic_oc", scalar3d_mid, kg / m2 / s, + grid_name); + + // Wet deposition of dust (bin1) [kg/m2/s] + add_field("wetdep_dust_bin1", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of dust (bin2) [kg/m2/s] + add_field("wetdep_dust_bin2", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of dust (bin3) [kg/m2/s] + add_field("wetdep_dust_bin3", scalar3d_mid, kg / m2 / s, grid_name); + + // Wet deposition of dust (bin4) [kg/m2/s] + add_field("wetdep_dust_bin4", scalar3d_mid, kg / m2 / s, grid_name); + + // Interstitial and cloudborne aerosol tracers of interest: mass (q) and + // number (n) mixing ratios + + // NOTE: Interstitial aerosols are updated in the interface using the + // "tendencies" from the wetscavenging process + + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { + // interstitial aerosol tracers of interest: number (n) mixing ratios + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(imode); + add_field(int_nmr_field_name, scalar3d_mid, n_unit, grid_name, + "tracers"); + + // cloudborne aerosol tracers of interest: number (n) mixing ratios + // Note: Do *not* add cld borne aerosols to the "tracer" group as these are + // not advected + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(imode); + + add_field(cld_nmr_field_name, scalar3d_mid, n_unit, grid_name); + + for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { + // (interstitial) aerosol tracers of interest: mass (q) mixing ratios + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(imode, ispec); + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } + + // (cloudborne) aerosol tracers of interest: mass (q) mixing ratios + // Note: Do *not* add cld borne aerosols to the "tracer" group as these + // are not advected + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(imode, ispec); + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_mid, q_unit, grid_name); + } + } + } + + // The following fields are not needed by this process but we define them so + // that we can create MAM4xx class objects like atmosphere, prognostics etc. + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_mid, q_unit, grid_name, + "tracers"); + } + + // ------------------------------------------------------------- + // These variables are "Computed" or outputs for the process + // ------------------------------------------------------------- + static constexpr auto m3 = m * m * m; + + // Aerosol dry particle diameter [m] + add_field("dgncur_a", scalar3d_mid_nmodes, m, grid_name); + + // Wet aerosol density [kg/m3] + add_field("wetdens", scalar3d_mid_nmodes, kg / m3, grid_name); + + // Aerosol water [kg/kg] + add_field("qaerwat", scalar3d_mid_nmodes, kg / kg, grid_name); + + // Wet aerosol diameter [m] + add_field("dgnumwet", scalar3d_mid_nmodes, m, grid_name); + + // Fraction of transported species that are insoluble [fraction] + add_field("fracis", scalar3d_mid, nondim, grid_name); + + // Aerosol wet deposition (interstitial) [kg/m2/s] + add_field("aerdepwetis", scalar2d_pconst, kg / m2 / s, grid_name); + + // Aerosol wet deposition (cloud water) [kg/m2/s] + add_field("aerdepwetcw", scalar2d_pconst, kg / m2 / s, grid_name); +} + +// ================================================================ +// INIT_BUFFERS +// ================================================================ +// ON HOST, initializeÑ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +void MAMWetscav::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); + + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMWetscav."); +} + +// ================================================================ +// INITIALIZE_IMPL +// ================================================================ +void MAMWetscav::initialize_impl(const RunType run_type) { + // --------------------------------------------------------------- + // Input fields read in from IC file, namelist or other processes + // --------------------------------------------------------------- + + // store fields only to be converted to dry mmrs in wet_atm_ + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + + // Following wet atm variables are NOT used by the process but we still + // need them to create atmosphere object + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + // Populate the dry atmosphere state with views from fields + // (NOTE: dry atmosphere has everything that wet + // atmosphere has along with z_surf, T_mid, p_mid, z_mid, z_iface, + // dz, p_del, cldfrac, w_updraft, pblh, phis and omega) + dry_atm_.z_surf = 0; + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_liq").get_view(); + + // The following dry_atm_ members *may* not be used by the process but they + // are needed for creating MAM4xx class objects like Atmosphere + dry_atm_.omega = get_field_in("omega").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.phis = get_field_in("phis").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.dz = buffer_.dz; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.w_updraft = buffer_.w_updraft; + + // ---- set wet/dry aerosol-related gas state data + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = get_field_out(mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // set wet/dry aerosol state data (interstitial aerosols only) + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { + const char *int_nmr_field_name = + mam_coupling::int_aero_nmr_field_name(imode); + wet_aero_.int_aero_nmr[imode] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[imode] = buffer_.dry_int_aero_nmr[imode]; + + const char *cld_nmr_field_name = + mam_coupling::cld_aero_nmr_field_name(imode); + wet_aero_.cld_aero_nmr[imode] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[imode] = wet_aero_.cld_aero_nmr[imode]; + + for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(imode, ispec); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[imode][ispec] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[imode][ispec] = + buffer_.dry_int_aero_mmr[imode][ispec]; + } + + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(imode, ispec); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[imode][ispec] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[imode][ispec] = + buffer_.dry_cld_aero_mmr[imode][ispec]; + } + } + } + + //--------------------------------------------------------------------------------- + // Allocate memory + //--------------------------------------------------------------------------------- + // Alllocate aerosol-related gas tendencies + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + dry_aero_tends_.gas_mmr[g] = view_2d("gas_mmr", ncol_, nlev_); + } + + // Allocate aerosol state tendencies (interstitial aerosols only) + for(int imode = 0; imode < mam_coupling::num_aero_modes(); ++imode) { + dry_aero_tends_.int_aero_nmr[imode] = view_2d("int_aero_nmr", ncol_, nlev_); + + for(int ispec = 0; ispec < mam_coupling::num_aero_species(); ++ispec) { + dry_aero_tends_.int_aero_mmr[imode][ispec] = + view_2d("int_aero_mmr", ncol_, nlev_); + } + } + + // Allocate work array + const int work_len = mam4::wetdep::get_aero_model_wetdep_work_len(); + work_ = view_2d("work", ncol_, work_len); + + // TODO: Following variables are from convective parameterization (not + // implemented yet in EAMxx), so should be zero for now + + sh_frac_ = view_2d("sh_frac", ncol_, nlev_); + Kokkos::deep_copy(sh_frac_, 0); + + // Deep convective cloud fraction [fraction] + dp_frac_ = view_2d("dp_frac", ncol_, nlev_); + Kokkos::deep_copy(dp_frac_, 0); + + // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] + evapcsh_ = view_2d("evapcsh", ncol_, nlev_); + Kokkos::deep_copy(evapcsh_, 0); + + // Evaporation rate of deep convective precipitation >=0. [kg/kg/s] + evapcdp_ = view_2d("evapcdp", ncol_, nlev_); + Kokkos::deep_copy(evapcdp_, 0); + + // Rain production, shallow convection [kg/kg/s] + rprdsh_ = view_2d("rprdsh", ncol_, nlev_); + Kokkos::deep_copy(rprdsh_, 0); + + // Rain production, deep convection [kg/kg/s] + rprddp_ = view_2d("rprddp", ncol_, nlev_); + Kokkos::deep_copy(rprddp_, 0); + + // In cloud water mixing ratio, deep convection + icwmrdp_ = view_2d("icwmrdp", ncol_, nlev_); + Kokkos::deep_copy(icwmrdp_, 0); + + // In cloud water mixing ratio, shallow convection + icwmrsh_ = view_2d("icwmrsh", ncol_, nlev_); + Kokkos::deep_copy(icwmrsh_, 0); + + // Detraining cld H20 from deep convection [kg/kg/s] + dlf_ = view_2d("dlf", ncol_, nlev_); + Kokkos::deep_copy(dlf_, 0); + + //--------------------------------------------------------------------------------- + // Setup preprocessing and post processing + //--------------------------------------------------------------------------------- + // set up our preprocess and postprocess functors + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); + + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); +} + +// ================================================================ +// RUN_IMPL +// ================================================================ +void MAMWetscav::run_impl(const double dt) { + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of all variables + // needed by this process or setting up MAM4xx classes and their objects + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; + const auto &dry_aero = dry_aero_; + const auto &work = work_; + const auto &dry_aero_tends = dry_aero_tends_; + + // --------------------------------------------------------------- + // These variables are "required" or pure inputs for the process + // --------------------------------------------------------------- + + //----------- Variables from convective scheme ------------- + + // TODO: Following variables are from convective parameterization (not + // implemented yet in EAMxx), so should be zero for now + + auto sh_frac = sh_frac_; + + // Deep convective cloud fraction [fraction] + auto dp_frac = dp_frac_; + + // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] + auto evapcsh = evapcsh_; + + // Evaporation rate of deep convective precipitation >=0. [kg/kg/s] + auto evapcdp = evapcdp_; + + // Rain production, shallow convection [kg/kg/s] + auto rprdsh = rprdsh_; + + // Rain production, deep convection [kg/kg/s] + auto rprddp = rprddp_; + + // In cloud water mixing ratio, deep convection + auto icwmrdp = icwmrdp_; + + // In cloud water mixing ratio, shallow convection + auto icwmrsh = icwmrsh_; + + // Detraining cld H20 from deep convection [kg/kg/s] + auto dlf = dlf_; + + //----------- Variables from macrophysics scheme ------------- + // Total cloud fraction + auto cldt = get_field_in("cldfrac_liq").get_view(); + + //----------- Variables from microphysics scheme ------------- + + // Evaporation from stratiform rain [kg/kg/s] + auto nevapr = get_field_in("nevapr").get_view(); + + // Stratiform rain production rate [kg/kg/s] + auto prain = get_field_in("precip_total_tend").get_view(); + // ------------------------------------------------------------------ + // These variables are "Computed" or pure outputs for the process + // ------------------------------------------------------------------ + + const auto aerdepwetis = get_field_out("aerdepwetis").get_view(); + const auto aerdepwetcw = get_field_out("aerdepwetcw").get_view(); + + const auto wet_geometric_mean_diameter_i = + get_field_out("dgnumwet").get_view(); + const auto dry_geometric_mean_diameter_i = + get_field_out("dgncur_a").get_view(); + const auto qaerwat = get_field_out("qaerwat").get_view(); + const auto wetdens = get_field_out("wetdens").get_view(); + + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + + // Making a local copy of 'nlev_' because we cannot use a member of a class + // inside a parallel_for. + const int nlev = nlev_; + + // Zero out tendencies otherwise, they are initialized to junk values + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + Kokkos::deep_copy(dry_aero_tends_.int_aero_nmr[m], 0); + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + Kokkos::deep_copy(dry_aero_tends_.int_aero_mmr[m][a], 0); + } + } + + // Loop over atmosphere columns + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const ThreadTeam &team) { + const int icol = team.league_rank(); // column index + + const auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); + // set surface state data + // fetch column-specific subviews into aerosol prognostics + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aero, icol); + // fetch column-specific subviews into aerosol tendencies + // Note: we are only updating interstitial aerosols. + mam4::Tendencies tends = + mam_coupling::interstitial_aerosols_tendencies_for_column( + dry_aero_tends, icol); + /// shallow_convective_precipitation_production + const auto rprdsh_icol = ekat::subview(rprdsh, icol); + // deep_convective_precipitation_production + const auto rprddp_icol = ekat::subview(rprddp, icol); + // deep_convective_precipitation_evaporation + const auto evapcdp_icol = ekat::subview(evapcdp, icol); + // shallow_convective_precipitation_evaporation = + const auto evapcsh_icol = ekat::subview(evapcsh, icol); + // deep_convective_cloud_fraction + const auto dp_frac_icol = ekat::subview(dp_frac, icol); + // shallow_convective_cloud_fraction = + const auto sh_frac_icol = ekat::subview(sh_frac, icol); + + const auto icwmrdp_col = ekat::subview(icwmrdp, icol); + const auto icwmrsh_icol = ekat::subview(icwmrsh, icol); + const auto nevapr_icol = ekat::subview(nevapr, icol); + const auto cldt_icol = ekat::subview(cldt, icol); + + const auto dlf_icol = ekat::subview(dlf, icol); + auto aerdepwetis_icol = ekat::subview(aerdepwetis, icol); + auto aerdepwetcw_icol = ekat::subview(aerdepwetcw, icol); + auto work_icol = ekat::subview(work, icol); + auto wet_diameter_icol = + ekat::subview(wet_geometric_mean_diameter_i, icol); + auto dry_diameter_icol = + ekat::subview(dry_geometric_mean_diameter_i, icol); + auto qaerwat_icol = ekat::subview(qaerwat, icol); + auto wetdens_icol = ekat::subview(wetdens, icol); + const auto prain_icol = ekat::subview(prain, icol); + + mam4::wetdep::aero_model_wetdep( + team, atm, progs, tends, dt, + // inputs + cldt_icol, rprdsh_icol, rprddp_icol, evapcdp_icol, evapcsh_icol, + dp_frac_icol, sh_frac_icol, icwmrdp_col, icwmrsh_icol, nevapr_icol, + dlf_icol, prain_icol, + // outputs + wet_diameter_icol, dry_diameter_icol, qaerwat_icol, wetdens_icol, + aerdepwetis_icol, aerdepwetcw_icol, work_icol); + team.team_barrier(); + // update interstitial aerosol state + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&](int kk) { + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const auto n_mode_i = progs.n_mode_i[m]; + const auto tends_n_mode_i = tends.n_mode_i[m]; + n_mode_i(kk) += tends_n_mode_i(kk) * dt; + for(int a = 0; a < mam4::num_species_mode(m); ++a) { + const auto q_aero_i = progs.q_aero_i[m][a]; + const auto tends_q_aero_i = tends.q_aero_i[m][a]; + q_aero_i(kk) += tends_q_aero_i(kk) * dt; + } + } + }); // parallel_for for update interstitial aerosol state + }); // icol parallel_for loop + + // call post processing to convert dry mixing ratios to wet mixing ratios + // and update the state + Kokkos::parallel_for("postprocess", scan_policy, postprocess_); + Kokkos::fence(); // wait before returning to calling function +} + +// ========================================================================================= +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp new file mode 100644 index 000000000000..9d1dc20bc88f --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_wetscav_process_interface.hpp @@ -0,0 +1,201 @@ +#ifndef EAMXX_MAM_WETSCAV_HPP +#define EAMXX_MAM_WETSCAV_HPP + +// For MAM4 aerosol configuration +#include + +// For declaring wetscav class derived from atm process class +#include "share/atm_process/atmosphere_process.hpp" + +// For component name +#include + +namespace scream { + +/* + * The class responsible to handle the aerosol wetscavenging + * + * The AD should store exactly ONE instance of this class stored + * in its list of subcomponents (the AD should make sure of this). + * + */ + +class MAMWetscav : public scream::AtmosphereProcess { + using KT = ekat::KokkosTypes; + using view_2d = typename KT::template view_2d; + + // a thread team dispatched to a single vertical column + using ThreadTeam = mam4::ThreadTeam; + + public: + // Constructor + MAMWetscav(const ekat::Comm &comm, const ekat::ParameterList ¶ms); + + // The type of subcomponent + AtmosphereProcessType type() const override { return AtmosphereProcessType::Physics; } + + // The name of the subcomponent + std::string name() const override { return "mam4_wetscav"; } + + // Set the grid and input output variables + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + // ON HOST, returns the number of bytes of device memory needed by the above + // Buffer type given the number of columns and vertical levels + size_t requested_buffer_size_in_bytes() const override { + return mam_coupling::buffer_size(ncol_, nlev_); + } + void init_buffers(const ATMBufferManager &buffer_manager) override; + + // Initialize variables + void initialize_impl(const RunType run_type) override; + + // Run the process by one time step + void run_impl(const double dt) override; + + // Finalize + void finalize_impl() override {/*Do nothing*/}; + + // Atmosphere processes often have a pre-processing step that constructs + // required variables from the set of fields stored in the field manager. + // This functor implements this step, which is called during run_impl. + struct Preprocess { + Preprocess() = default; + + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol_in, const int nlev_in, + const mam_coupling::WetAtmosphere &wet_atm_in, + const mam_coupling::AerosolState &wet_aero_in, + const mam_coupling::DryAtmosphere &dry_atm_in, + const mam_coupling::AerosolState &dry_aero_in) { + ncol_pre_ = ncol_in; + nlev_pre_ = nlev_in; + wet_atm_pre_ = wet_atm_in; + wet_aero_pre_ = wet_aero_in; + dry_atm_pre_ = dry_atm_in; + dry_aero_pre_ = dry_aero_in; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + // first, compute dry fields + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, i); + compute_dry_mixing_ratios(team, wet_atm_pre_, wet_aero_pre_, + dry_aero_pre_, i); + team.team_barrier(); + // second, we can use dry fields to compute dz, zmin, zint + compute_vertical_layer_heights(team, dry_atm_pre_, i); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, i); + // allows kernels below to use layer heights operator() + team.team_barrier(); + } + + // Number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + + // Local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + mam_coupling::AerosolState wet_aero_pre_, dry_aero_pre_; + }; // MAMWetscav::Preprocess + + // Postprocessing functor + struct Postprocess { + Postprocess() = default; + + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_post_ = ncol; + nlev_post_ = nlev; + wet_atm_post_ = wet_atm; + wet_aero_post_ = wet_aero; + dry_atm_post_ = dry_atm; + dry_aero_post_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_post_, dry_aero_post_, + wet_aero_post_, i); + team.team_barrier(); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_post_, nlev_post_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_post_; + mam_coupling::DryAtmosphere dry_atm_post_; + mam_coupling::AerosolState wet_aero_post_, dry_aero_post_; + }; // MAMWetscav::Postprocess + + private: + // ----------------------------------------------- + // Local variables + // ------------------------------------------------ + // pre- and postprocessing scratch pads (for wet <-> dry conversions) + Preprocess preprocess_; + Postprocess postprocess_; + + // Number of horizontal columns and vertical levels + int ncol_, nlev_; + + // Number of aerosol modes + static constexpr int ntot_amode_ = mam4::AeroConfig::num_modes(); + + // Atmospheric variables + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + + // Work arrays + view_2d work_; + + // TODO: Following variables are from convective parameterization (not + // implemented yet in EAMxx), so should be zero for now + + view_2d sh_frac_; + + // Deep convective cloud fraction [fraction] + view_2d dp_frac_; + + // Evaporation rate of shallow convective precipitation >=0. [kg/kg/s] + view_2d evapcsh_; + + view_2d evapcdp_; + + // Rain production, shallow convection [kg/kg/s] + view_2d rprdsh_; + + // Rain production, deep convection [kg/kg/s] + view_2d rprddp_; + + // In cloud water mixing ratio, deep convection + view_2d icwmrdp_; + + // In cloud water mixing ratio, shallow convection + view_2d icwmrsh_; + + // Detraining cld H20 from deep convection [kg/kg/s] + view_2d dlf_; + + // Aerosol states + mam_coupling::AerosolState wet_aero_, dry_aero_, dry_aero_tends_; + + mam_coupling::Buffer buffer_; + + std::shared_ptr m_grid; +}; // class MAMWetscav + +} // namespace scream + +#endif // EAMXX_MAM_WETSCAV_HPP diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp index 3a0011468826..a3a5f746aa30 100644 --- a/components/eamxx/src/physics/mam/mam_coupling.hpp +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -562,6 +562,34 @@ mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, return progs; } + +// Given an AerosolState with views for dry aerosol quantities, creates a +// mam4::Tendencies object for the column with the given index with +// ONLY INTERSTITIAL AEROSOL VIEWS DEFINED. This object can be provided to +// mam4xx for the column. +KOKKOS_INLINE_FUNCTION +mam4::Tendencies interstitial_aerosols_tendencies_for_column(const AerosolState& dry_aero, + const int column_index) { + constexpr int nlev = mam4::nlev; + mam4::Tendencies tends(nlev); + for (int m = 0; m < num_aero_modes(); ++m) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), + "int_aero_nmr not defined for dry aerosol state!"); + tends.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.int_aero_mmr[m][a].data()) { + tends.q_aero_i[m][a] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + } + } + } + for (int g = 0; g < num_aero_gases(); ++g) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.gas_mmr[g].data(), + "gas_mmr not defined for dry aerosol state!"); + tends.q_gas[g] = ekat::subview(dry_aero.gas_mmr[g], column_index); + } + return tends; +} + // Given a dry aerosol state, creates a mam4::Prognostics object for the column // with the given index with interstitial and cloudborne aerosol views defined. // This object can be provided to mam4xx for the column. @@ -581,7 +609,25 @@ mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, } return progs; } - +// Given a dry aerosol state tendencies, creates a mam4::Tendencies object for the column +// with the given index with interstitial and cloudborne aerosol views defined. +// This object can be provided to mam4xx for the column. +KOKKOS_INLINE_FUNCTION +mam4::Tendencies aerosols_tendencies_for_column(const AerosolState& dry_aero, + const int column_index) { + auto tends = interstitial_aerosols_tendencies_for_column(dry_aero, column_index); + for (int m = 0; m < num_aero_modes(); ++m) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.cld_aero_nmr[m].data(), + "Tendencies : dry_cld_aero_nmr not defined for aerosol state!"); + tends.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.cld_aero_mmr[m][a].data()) { + tends.q_aero_c[m][a] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + } + } + } + return tends; +} // Given a thread team and a dry atmosphere state, dispatches threads from the // team to compute vertical layer heights and interfaces for the column with // the given index. @@ -592,21 +638,27 @@ void compute_vertical_layer_heights(const Team& team, EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), "Given column index does not correspond to given team!"); + //outputs const auto dz = ekat::subview(dry_atm.dz, column_index); const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); + //inputs const auto qv = ekat::subview(dry_atm.qv, column_index); const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); + // NOTE: we are using dry qv. Does calculate_dz require dry or wet? PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, // inputs dz);//output team.team_barrier(); + // NOTE: we are not currently allowing surface topography: + EKAT_KERNEL_ASSERT_MSG(dry_atm.z_surf == 0, "dry_atm.z_surf must be zero"); PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, //inputs z_iface); //output team.team_barrier(); // likely necessary to have z_iface up to date - PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); + PF::calculate_z_mid(team, mam4::nlev, z_iface, //input + z_mid); //output } // Given a thread team and wet and dry atmospheres, dispatches threads from the @@ -724,6 +776,24 @@ void compute_wet_mixing_ratios(const Team& team, }); } +// Computes the reciprocal of pseudo density for a column +inline +void compute_recipical_pseudo_density(haero::ThreadTeamPolicy team_policy, + const_view_2d pdel, + const int nlev, + // output + view_2d rpdel) { + Kokkos::parallel_for( + team_policy, KOKKOS_LAMBDA(const haero::ThreadTeam &team) { + const int icol = team.league_rank(); + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, 0, nlev), [&](int kk) { + EKAT_KERNEL_ASSERT_MSG(0 < pdel(icol, kk), + "Error: pdel should be > 0.\n"); + rpdel(icol, kk) = 1 / pdel(icol, kk); + }); + }); +} // Scream (or EAMxx) can sometimes extend views beyond model levels (nlev) as it uses // "packs". Following function copies a 2d view till model levels inline diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp new file mode 100644 index 000000000000..29aaca421ea9 --- /dev/null +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -0,0 +1,138 @@ +#ifndef SRF_EMISSION_HPP +#define SRF_EMISSION_HPP + +#include "share/util/scream_timing.hpp" + +namespace scream::mam_coupling { +namespace { + +template +struct srfEmissFunctions { + using Device = DeviceType; + + using KT = KokkosTypes; + using MemberType = typename KT::MemberType; + + struct srfEmissTimeState { + srfEmissTimeState() = default; + // Whether the timestate has been initialized. + // The current month + int current_month = -1; + // Julian Date for the beginning of the month, as defined in + // /src/share/util/scream_time_stamp.hpp + // See this file for definition of Julian Date. + Real t_beg_month; + // Current simulation Julian Date + Real t_now; + // Number of days in the current month, cast as a Real + Real days_this_month; + }; // srfEmissTimeState + + struct srfEmissData { + srfEmissData() = default; + srfEmissData(const int ncol_, const int nsectors_) + : ncols(ncol_), nsectors(nsectors_) { + init(ncols, nsectors, true); + } + + void init(const int ncol, const int nsector, const bool allocate) { + ncols = ncol; + nsectors = nsector; + if(allocate) emiss_sectors = view_2d("AllSectors", nsectors, ncols); + } // srfEmissData init + + // Basic spatial dimensions of the data + int ncols, nsectors; + view_2d emiss_sectors; + }; // srfEmissData + + struct srfEmissInput { + srfEmissInput() = default; + srfEmissInput(const int ncols_, const int nsectors_) { + init(ncols_, nsectors_); + } + + void init(const int ncols_, const int nsectors_) { + data.init(ncols_, nsectors_, true); + } + srfEmissData data; // All srfEmiss fields + }; // srfEmissInput + + // The output is really just srfEmissData, but for clarity it might + // help to see a srfEmissOutput along a srfEmissInput in functions signatures + using srfEmissOutput = srfEmissData; + + /* ------------------------------------------------------------------------------------------- + */ + // Surface emissions routines + template + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &srfEmiss_data_file, + const std::array &field_names, + const std::string &map_file); + + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &srfEmiss_data_file, + const std::vector &field_names, const std::string &map_file); + + static std::shared_ptr create_srfEmiss_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &srfEmiss_data_file); + + static void srfEmiss_main(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissOutput &data_out); + + static void update_srfEmiss_data_from_file( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, + const int time_index, // zero-based + AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input); + static void update_srfEmiss_timestate( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, AbstractRemapper &srfEmiss_horiz_interp, + srfEmissTimeState &time_state, srfEmissInput &srfEmiss_beg, + srfEmissInput &srfEmiss_end); + + // The following three are called during srfEmiss_main + static void perform_time_interpolation(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissOutput &data_out); + + // Performs convex interpolation of x0 and x1 at point t + template + KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, + const ScalarX &x1, + const ScalarT &t); + template + static void init_srf_emiss_objects( + const int ncol, const std::shared_ptr &grid, + const std::string &data_file, + const std::array §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader); + + static void init_srf_emiss_objects( + const int ncol, const std::shared_ptr &grid, + const std::string &data_file, const std::vector §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader); + +}; // struct srfEmissFunctions +} // namespace +} // namespace scream::mam_coupling +#endif // SRF_EMISSION_HPP + +#include "srf_emission_impl.hpp" diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp new file mode 100644 index 000000000000..64ae65714c59 --- /dev/null +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -0,0 +1,293 @@ +#ifndef SRF_EMISSION_IMPL_HPP +#define SRF_EMISSION_IMPL_HPP + +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +namespace scream::mam_coupling { +namespace { + +template +std::shared_ptr +srfEmissFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::vector §or_names, + const std::string &map_file) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, "ncol"); + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("srf_emiss_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + // if the file's grid is same as model's native grid, we identity remapper + // (i.e., no interpolation) + if(ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen srfEmiss data to fit " + "the model. We only allow\n" + "srfEmiss data to be at the same or coarser resolution as " + "the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG( + map_file != "", + "ERROR: srfEmiss data is on a different grid than the model one,\n" + "but srfEmiss_remap_file is missing from srfEmiss parameter " + "list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto nondim = ekat::units::Units::nondimensional(); + + std::vector field_emiss_sectors; + + const int sector_size = sector_names.size(); + for(int icomp = 0; icomp < sector_size; ++icomp) { + auto comp_name = sector_names[icomp]; + // set and allocate fields + Field f(FieldIdentifier(comp_name, layout_2d, nondim, tgt_grid->name())); + f.allocate_view(); + field_emiss_sectors.push_back(f); + remapper->register_field_from_tgt(f); + } + + remapper->registration_ends(); + + return remapper; +} // create_horiz_remapper + +template +std::shared_ptr +srfEmissFunctions::create_srfEmiss_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &srfEmiss_data_file) { + std::vector field_emiss_sectors; + for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { + field_emiss_sectors.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(srfEmiss_data_file, io_grid, + field_emiss_sectors, true); +} // create_srfEmiss_data_reader + +template +template +KOKKOS_INLINE_FUNCTION +ScalarX srfEmissFunctions::linear_interp(const ScalarX &x0, + const ScalarX &x1, + const ScalarT &t) { + return (1 - t) * x0 + t * x1; +} // linear_interp + +template +void srfEmissFunctions::perform_time_interpolation( + const srfEmissTimeState &time_state, const srfEmissInput &data_beg, + const srfEmissInput &data_end, const srfEmissOutput &data_out) { + // NOTE: we *assume* data_beg and data_end have the *same* hybrid v coords. + // IF this ever ceases to be the case, you can interp those too. + + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + + // Gather time stamp info + auto &t_now = time_state.t_now; + auto &t_beg = time_state.t_beg_month; + auto &delta_t = time_state.days_this_month; + + // At this stage, begin/end must have the same dimensions + EKAT_REQUIRE(data_end.data.ncols == data_beg.data.ncols); + + auto delta_t_fraction = (t_now - t_beg) / delta_t; + + EKAT_REQUIRE_MSG(delta_t_fraction >= 0 && delta_t_fraction <= 1, + "Error! Convex interpolation with coefficient out of " + "[0,1].\n t_now : " + + std::to_string(t_now) + + "\n" + " t_beg : " + + std::to_string(t_beg) + + "\n delta_t: " + std::to_string(delta_t) + "\n"); + + const int nsectors = data_beg.data.nsectors; + const int ncols = data_beg.data.ncols; + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + const auto policy = ESU::get_default_team_policy(ncols, nsectors); + + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MemberType &team) { + const int icol = team.league_rank(); // column index + Real accum = 0; + // Parallel reduction over sectors + // FIXME: Do we need to use Kokkos::Single for each team here??? + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, nsectors), + [&](const int i, Real &update) { + const auto beg = data_beg.data.emiss_sectors(i, icol); + const auto end = data_end.data.emiss_sectors(i, icol); + update += linear_interp(beg, end, delta_t_fraction); + }, + accum); + // Assign the accumulated value to the output + data_out.emiss_sectors(0, icol) = accum; + }); + Kokkos::fence(); +} // perform_time_interpolation + +template +void srfEmissFunctions::srfEmiss_main(const srfEmissTimeState &time_state, + const srfEmissInput &data_beg, + const srfEmissInput &data_end, + const srfEmissOutput &data_out) { + // Beg/End/Tmp month must have all sizes matching + + EKAT_REQUIRE_MSG( + data_end.data.ncols == data_beg.data.ncols, + "Error! srfEmissInput data structs must have the same number of " + "columns/levels.\n"); + + // Horiz interpolation can be expensive, and does not depend on the particular + // time of the month, so it can be done ONCE per month, *outside* + // srfEmiss_main (when updating the beg/end states, reading them from file). + EKAT_REQUIRE_MSG( + data_end.data.ncols == data_out.ncols, + "Error! Horizontal interpolation is performed *before* " + "calling srfEmiss_main,\n" + " srfEmissInput and srfEmissOutput data structs must have the " + "same number columns.\n"); + + // Step 1. Perform time interpolation + perform_time_interpolation(time_state, data_beg, data_end, data_out); +} // srfEmiss_main + +template +void srfEmissFunctions::update_srfEmiss_data_from_file( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + const int time_index, // zero-based + AbstractRemapper &srfEmiss_horiz_interp, srfEmissInput &srfEmiss_input) { + using namespace ShortFieldTagsNames; + using ESU = ekat::ExeSpaceUtils; + using Member = typename KokkosTypes::MemberType; + + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); + + // 1. Read from file + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::read_data"); + scorpio_reader->read_variables(time_index); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::read_data"); + + // 2. Run the horiz remapper (it is a do-nothing op if srfEmiss data is on + // same grid as model) + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::horiz_remap"); + srfEmiss_horiz_interp.remap(/*forward = */ true); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::horiz_remap"); + + // 3. Copy from the tgt field of the remapper into the srfEmiss_data, padding + // data if necessary + start_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); + // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, + // tau_sw, tau_lw + + const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) + .get_header() + .get_identifier() + .get_layout(); + + const int ncols = layout.dim(COL); + + // Read fields from the file + for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { + auto sector = + srfEmiss_horiz_interp.get_tgt_field(i).get_view(); + const auto emiss = + Kokkos::subview(srfEmiss_input.data.emiss_sectors, i, Kokkos::ALL()); + Kokkos::deep_copy(emiss, sector); + } + + Kokkos::fence(); + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file::copy_and_pad"); + + stop_timer("EAMxx::srfEmiss::update_srfEmiss_data_from_file"); + +} // END update_srfEmiss_data_from_file + +template +void srfEmissFunctions::update_srfEmiss_timestate( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + AbstractRemapper &srfEmiss_horiz_interp, srfEmissTimeState &time_state, + srfEmissInput &srfEmiss_beg, srfEmissInput &srfEmiss_end) { + // Now we check if we have to update the data that changes monthly + // NOTE: This means that srfEmiss assumes monthly data to update. Not + // any other frequency. + const auto month = ts.get_month() - 1; // Make it 0-based + if(month != time_state.current_month) { + // Update the srfEmiss time state information + time_state.current_month = month; + time_state.t_beg_month = + util::TimeStamp({ts.get_year(), month + 1, 1}, {0, 0, 0}) + .frac_of_year_in_days(); + time_state.days_this_month = util::days_in_month(ts.get_year(), month + 1); + + // Copy srfEmiss_end'data into srfEmiss_beg'data, and read in the new + // srfEmiss_end + std::swap(srfEmiss_beg, srfEmiss_end); + + // Update the srfEmiss forcing data for this month and next month + // Start by copying next months data to this months data structure. + // NOTE: If the timestep is bigger than monthly this could cause the wrong + // values + // to be assigned. A timestep greater than a month is very unlikely + // so we will proceed. + int next_month = (time_state.current_month + 1) % 12; + update_srfEmiss_data_from_file(scorpio_reader, ts, next_month, + srfEmiss_horiz_interp, srfEmiss_end); + } + +} // END updata_srfEmiss_timestate + +template +void srfEmissFunctions::init_srf_emiss_objects( + const int ncol, const std::shared_ptr &grid, + const std::string &data_file, const std::vector §ors, + const std::string &srf_map_file, + // output + std::shared_ptr &SrfEmissHorizInterp, + srfEmissInput &SrfEmissData_start, srfEmissInput &SrfEmissData_end, + srfEmissOutput &SrfEmissData_out, + std::shared_ptr &SrfEmissDataReader) { + // Init horizontal remap + SrfEmissHorizInterp = + create_horiz_remapper(grid, data_file, sectors, srf_map_file); + + // Initialize the size of start/end/out data structures + SrfEmissData_start = srfEmissInput(ncol, sectors.size()); + SrfEmissData_end = srfEmissInput(ncol, sectors.size()); + SrfEmissData_out.init(ncol, 1, true); + + // Create reader (an AtmosphereInput object) + SrfEmissDataReader = + create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); +} // init_srf_emiss_objects + +} // namespace +} // namespace scream::mam_coupling + +#endif // SRF_EMISSION_IMPL_HPP diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp index 3af33eaea164..8ebfc9c14c01 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp @@ -4,6 +4,7 @@ #include "share/grid/remap/refining_remapper_p2p.hpp" #include "share/grid/remap/do_nothing_remapper.hpp" #include "share/util/scream_utils.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include #include diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp index 51773cb249df..a5a158f44a17 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp @@ -51,7 +51,7 @@ TEST_CASE("nudging_tests") { const int nlevs_fine = 2*nlevs_data -1; // Files names - auto postfix = ".INSTANT.nsteps_x1." + get_t0().to_string() + ".nc"; + auto postfix = ".INSTANT.nsteps_x1.np*." + get_t0().to_string() + ".nc"; auto nudging_data = "nudging_data" + postfix; auto nudging_data_filled = "nudging_data_filled" + postfix; auto map_file = "map_ncol" + std::to_string(ngcols_data) diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index 1a01f607c630..a1dfc946267d 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -45,7 +45,6 @@ if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos eti/p3_ice_relaxation_timescale.cpp eti/p3_ice_nucleation.cpp eti/p3_ice_cldliq_wet_growth.cpp - eti/p3_get_latent_heat.cpp eti/p3_check_values.cpp eti/p3_incloud_mixingratios.cpp eti/p3_subgrid_variance_scaling.cpp @@ -63,25 +62,25 @@ endif() # List of dispatch source files if monolithic kernels are off set(P3_SK_SRCS - disp/p3_check_values_impl_disp.cpp - disp/p3_ice_sed_impl_disp.cpp - disp/p3_main_impl_part1_disp.cpp + disp/p3_check_values_impl_disp.cpp + disp/p3_ice_sed_impl_disp.cpp + disp/p3_main_impl_part1_disp.cpp disp/p3_main_impl_part3_disp.cpp - disp/p3_cloud_sed_impl_disp.cpp - disp/p3_main_impl_disp.cpp - disp/p3_main_impl_part2_disp.cpp + disp/p3_cloud_sed_impl_disp.cpp + disp/p3_main_impl_disp.cpp + disp/p3_main_impl_part2_disp.cpp disp/p3_rain_sed_impl_disp.cpp ) set(P3_LIBS "p3") -if (SCREAM_SMALL_KERNELS) +if (SCREAM_P3_SMALL_KERNELS) add_library(p3 ${P3_SRCS} ${P3_SK_SRCS}) else() add_library(p3 ${P3_SRCS}) if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(p3_sk ${P3_SRCS} ${P3_SK_SRCS}) - # Always build shoc_sk with SCREAM_SMALL_KERNELS on - target_compile_definitions(p3_sk PUBLIC "SCREAM_SMALL_KERNELS") + # Always build p3_sk with SCREAM_P3_SMALL_KERNELS on + target_compile_definitions(p3_sk PUBLIC "SCREAM_P3_SMALL_KERNELS") list(APPEND P3_LIBS "p3_sk") endif() endif() diff --git a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp index 0d2aad5247dd..f1e84750c250 100644 --- a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp @@ -46,8 +46,8 @@ ::ice_sedimentation_disp( // Ice sedimentation: (adaptive substepping) ice_sedimentation( - ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(cld_frac_i, i), - ekat::subview(inv_dz, i), team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, + ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(cld_frac_i, i), + ekat::subview(inv_dz, i), team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, ekat::subview(qi, i), ekat::subview(qi_incld, i), ekat::subview(ni, i), ekat::subview(ni_incld, i), ekat::subview(qm, i), ekat::subview(qm_incld, i), ekat::subview(bm, i), ekat::subview(bm_incld, i), ekat::subview(qi_tend, i), ekat::subview(ni_tend, i), ice_table_vals, precip_ice_surf(i), p3constants); @@ -60,7 +60,6 @@ void Functions ::homogeneous_freezing_disp( const uview_2d& T_atm, const uview_2d& inv_exner, - const uview_2d& latent_heat_fusion, const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_2d& qc, const uview_2d& nc, @@ -89,7 +88,7 @@ ::homogeneous_freezing_disp( // homogeneous freezing of cloud and rain homogeneous_freezing( - ekat::subview(T_atm, i), ekat::subview(inv_exner, i), ekat::subview(latent_heat_fusion, i), team, nk, ktop, kbot, kdir, + ekat::subview(T_atm, i), ekat::subview(inv_exner, i), team, nk, ktop, kbot, kdir, ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(th_atm, i)); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp index c3692f3f814f..b79b4f6d072c 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp @@ -32,16 +32,16 @@ ::p3_main_init_disp( const uview_2d& qv_supersat_i, const uview_2d& qtend_ignore, const uview_2d& ntend_ignore, const uview_2d& mu_c, const uview_2d& lamc, const uview_2d& rho_qi, const uview_2d& qv2qi_depos_tend, const uview_2d& precip_total_tend, const uview_2d& nevapr, const uview_2d& precip_liq_flux, const uview_2d& precip_ice_flux) -{ +{ using ExeSpace = typename KT::ExeSpace; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); Kokkos::parallel_for("p3_main_init", policy, KOKKOS_LAMBDA(const MemberType& team) { - - const Int i = team.league_rank(); - precip_liq_surf(i) = 0; - precip_ice_surf(i) = 0; + + const Int i = team.league_rank(); + precip_liq_surf(i) = 0; + precip_ice_surf(i) = 0; Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nk_pack), [&] (Int k) { @@ -107,6 +107,7 @@ ::p3_main_internal_disp( const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, + const P3Temporaries& temporaries, const WorkspaceManager& workspace_mgr, Int nj, Int nk, @@ -114,10 +115,6 @@ ::p3_main_internal_disp( { using ExeSpace = typename KT::ExeSpace; - view_2d latent_heat_sublim("latent_heat_sublim", nj, nk), latent_heat_vapor("latent_heat_vapor", nj, nk), latent_heat_fusion("latent_heat_fusion", nj, nk); - - get_latent_heat(nj, nk, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion); - const Int nk_pack = ekat::npack(nk); // load constants into local vars @@ -131,85 +128,99 @@ ::p3_main_internal_disp( view_1d nucleationPossible("nucleationPossible", nj); view_1d hydrometeorsPresent("hydrometeorsPresent", nj); - // - // Create temporary variables needed for p3 - // - view_2d - mu_r("mu_r", nj, nk_pack), // shape parameter of rain - T_atm("T_atm", nj, nk_pack), // temperature at the beginning of the microphysics step [K] - - // 2D size distribution and fallspeed parameters - lamr("lamr", nj, nk_pack), logn0r("logn0r", nj, nk_pack), nu("nu", nj, nk_pack), - cdist("cdist", nj, nk_pack), cdist1("cdist1", nj, nk_pack), cdistr("cdistr", nj, nk_pack), - - // Variables needed for in-cloud calculations - // Inverse cloud fractions (1/cld) - inv_cld_frac_i("inv_cld_frac_i", nj, nk_pack), inv_cld_frac_l("inv_cld_frac_l", nj, nk_pack), inv_cld_frac_r("inv_cld_frac_r", nj, nk_pack), - // In cloud mass-mixing ratios - qc_incld("qc_incld", nj, nk_pack), qr_incld("qr_incld", nj, nk_pack), qi_incld("qi_incld", nj, nk_pack), qm_incld("qm_incld", nj, nk_pack), - // In cloud number concentrations - nc_incld("nc_incld", nj, nk_pack), nr_incld("nr_incld", nj, nk_pack), ni_incld("ni_incld", nj, nk_pack), bm_incld("bm_incld", nj, nk_pack), - - // Other - inv_dz("inv_dz", nj, nk_pack), inv_rho("inv_rho", nj, nk_pack), ze_ice("ze_ice", nj, nk_pack), ze_rain("ze_rain", nj, nk_pack), - prec("prec", nj, nk_pack), rho("rho", nj, nk_pack), rhofacr("rhofacr", nj, nk_pack), rhofaci("rhofaci", nj, nk_pack), - acn("acn", nj, nk_pack), qv_sat_l("qv_sat", nj, nk_pack), qv_sat_i("qv_sat_i", nj, nk_pack), sup("sup", nj, nk_pack), - qv_supersat_i("qv_supersat", nj, nk_pack), tmparr2("tmparr2", nj, nk_pack), exner("exner", nj, nk_pack), - diag_equiv_reflectivity("diag_equiv_ref", nj, nk_pack), diag_vm_qi("diag_vm_qi", nj, nk_pack), diag_diam_qi("diag_diam_qi", nj, nk_pack), - pratot("pratot", nj, nk_pack), prctot("prctot", nj, nk_pack), - - // p3_tend_out, may not need these - qtend_ignore("qtend_ignore", nj, nk_pack), ntend_ignore("ntend_ignore", nj, nk_pack), - - // Variables still used in F90 but removed from C++ interface - mu_c("mu_c", nj, nk_pack), lamc("lamc", nj, nk_pack), precip_total_tend("precip_total_tend", nj, nk_pack), - nevapr("nevapr", nj, nk_pack), qr_evap_tend("qr_evap_tend", nj, nk_pack), - - // cloud sedimentation - v_qc("v_qc", nj, nk_pack), v_nc("v_nc", nj, nk_pack), flux_qx("flux_qx", nj, nk_pack), flux_nx("flux_nx", nj, nk_pack), - - // ice sedimentation - v_qit("v_qit", nj, nk_pack), v_nit("v_nit", nj, nk_pack), flux_nit("flux_nit", nj, nk_pack), flux_bir("flux_bir", nj, nk_pack), - flux_qir("flux_qir", nj, nk_pack), flux_qit("flux_qit", nj, nk_pack), - - // rain sedimentation - v_qr("v_qr", nj, nk_pack), v_nr("v_nr", nj, nk_pack); - // Get views of all inputs - auto pres = diagnostic_inputs.pres; - auto dz = diagnostic_inputs.dz; - auto nc_nuceat_tend = diagnostic_inputs.nc_nuceat_tend; - auto nccn_prescribed = diagnostic_inputs.nccn; - auto ni_activated = diagnostic_inputs.ni_activated; - auto inv_qc_relvar = diagnostic_inputs.inv_qc_relvar; - auto dpres = diagnostic_inputs.dpres; - auto inv_exner = diagnostic_inputs.inv_exner; - auto cld_frac_i = diagnostic_inputs.cld_frac_i; - auto cld_frac_l = diagnostic_inputs.cld_frac_l; - auto cld_frac_r = diagnostic_inputs.cld_frac_r; - auto col_location = infrastructure.col_location; - auto qc = prognostic_state.qc; - auto nc = prognostic_state.nc; - auto qr = prognostic_state.qr; - auto nr = prognostic_state.nr; - auto qi = prognostic_state.qi; - auto qm = prognostic_state.qm; - auto ni = prognostic_state.ni; - auto bm = prognostic_state.bm; - auto qv = prognostic_state.qv; - auto th = prognostic_state.th; - auto diag_eff_radius_qc = diagnostic_outputs.diag_eff_radius_qc; - auto diag_eff_radius_qi = diagnostic_outputs.diag_eff_radius_qi; - auto diag_eff_radius_qr = diagnostic_outputs.diag_eff_radius_qr; - auto qv2qi_depos_tend = diagnostic_outputs.qv2qi_depos_tend; - auto rho_qi = diagnostic_outputs.rho_qi; - auto precip_liq_flux = diagnostic_outputs.precip_liq_flux; - auto precip_ice_flux = diagnostic_outputs.precip_ice_flux; - auto qv_prev = diagnostic_inputs.qv_prev; - auto t_prev = diagnostic_inputs.t_prev; - auto liq_ice_exchange = history_only.liq_ice_exchange; - auto vap_liq_exchange = history_only.vap_liq_exchange; - auto vap_ice_exchange = history_only.vap_ice_exchange; + auto pres = diagnostic_inputs.pres; + auto dz = diagnostic_inputs.dz; + auto nc_nuceat_tend = diagnostic_inputs.nc_nuceat_tend; + auto nccn_prescribed = diagnostic_inputs.nccn; + auto ni_activated = diagnostic_inputs.ni_activated; + auto inv_qc_relvar = diagnostic_inputs.inv_qc_relvar; + auto dpres = diagnostic_inputs.dpres; + auto inv_exner = diagnostic_inputs.inv_exner; + auto cld_frac_i = diagnostic_inputs.cld_frac_i; + auto cld_frac_l = diagnostic_inputs.cld_frac_l; + auto cld_frac_r = diagnostic_inputs.cld_frac_r; + auto col_location = infrastructure.col_location; + auto qc = prognostic_state.qc; + auto nc = prognostic_state.nc; + auto qr = prognostic_state.qr; + auto nr = prognostic_state.nr; + auto qi = prognostic_state.qi; + auto qm = prognostic_state.qm; + auto ni = prognostic_state.ni; + auto bm = prognostic_state.bm; + auto qv = prognostic_state.qv; + auto th = prognostic_state.th; + auto diag_eff_radius_qc = diagnostic_outputs.diag_eff_radius_qc; + auto diag_eff_radius_qi = diagnostic_outputs.diag_eff_radius_qi; + auto diag_eff_radius_qr = diagnostic_outputs.diag_eff_radius_qr; + auto qv2qi_depos_tend = diagnostic_outputs.qv2qi_depos_tend; + auto rho_qi = diagnostic_outputs.rho_qi; + auto precip_liq_flux = diagnostic_outputs.precip_liq_flux; + auto precip_ice_flux = diagnostic_outputs.precip_ice_flux; + auto precip_total_tend = diagnostic_outputs.precip_total_tend; + auto nevapr = diagnostic_outputs.nevapr; + auto qv_prev = diagnostic_inputs.qv_prev; + auto t_prev = diagnostic_inputs.t_prev; + auto liq_ice_exchange = history_only.liq_ice_exchange; + auto vap_liq_exchange = history_only.vap_liq_exchange; + auto vap_ice_exchange = history_only.vap_ice_exchange; + auto mu_r = temporaries.mu_r; + auto T_atm = temporaries.T_atm; + auto lamr = temporaries.lamr; + auto logn0r = temporaries.logn0r; + auto nu = temporaries.nu; + auto cdist = temporaries.cdist; + auto cdist1 = temporaries.cdist1; + auto cdistr = temporaries.cdistr; + auto inv_cld_frac_i = temporaries.inv_cld_frac_i; + auto inv_cld_frac_l = temporaries.inv_cld_frac_l; + auto inv_cld_frac_r = temporaries.inv_cld_frac_r; + auto qc_incld = temporaries.qc_incld; + auto qr_incld = temporaries.qr_incld; + auto qi_incld = temporaries.qi_incld; + auto qm_incld = temporaries.qm_incld; + auto nc_incld = temporaries.nc_incld; + auto nr_incld = temporaries.nr_incld; + auto ni_incld = temporaries.ni_incld; + auto bm_incld = temporaries.bm_incld; + auto inv_dz = temporaries.inv_dz; + auto inv_rho = temporaries.inv_rho; + auto ze_ice = temporaries.ze_ice; + auto ze_rain = temporaries.ze_rain; + auto prec = temporaries.prec; + auto rho = temporaries.rho; + auto rhofacr = temporaries.rhofacr; + auto rhofaci = temporaries.rhofaci; + auto acn = temporaries.acn; + auto qv_sat_l = temporaries.qv_sat_l; + auto qv_sat_i = temporaries.qv_sat_i; + auto sup = temporaries.sup; + auto qv_supersat_i = temporaries.qv_supersat_i; + auto tmparr2 = temporaries.tmparr2; + auto exner = temporaries.exner; + auto diag_equiv_reflectivity = temporaries.diag_equiv_reflectivity; + auto diag_vm_qi = temporaries.diag_vm_qi; + auto diag_diam_qi = temporaries.diag_diam_qi; + auto pratot = temporaries.pratot; + auto prctot = temporaries.prctot; + auto qtend_ignore = temporaries.qtend_ignore; + auto ntend_ignore = temporaries.ntend_ignore; + auto mu_c = temporaries.mu_c; + auto lamc = temporaries.lamc; + auto qr_evap_tend = temporaries.qr_evap_tend; + auto v_qc = temporaries.v_qc; + auto v_nc = temporaries.v_nc; + auto flux_qx = temporaries.flux_qx; + auto flux_nx = temporaries.flux_nx; + auto v_qit = temporaries.v_qit; + auto v_nit = temporaries.v_nit; + auto flux_nit = temporaries.flux_nit; + auto flux_bir = temporaries.flux_bir; + auto flux_qir = temporaries.flux_qir; + auto flux_qit = temporaries.flux_qit; + auto v_qr = temporaries.v_qr; + auto v_nr = temporaries.v_nr; // we do not want to measure init stuff auto start = std::chrono::steady_clock::now(); @@ -229,7 +240,7 @@ ::p3_main_internal_disp( p3_main_part1_disp( nj, nk, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, pres, dpres, dz, nc_nuceat_tend, nccn_prescribed, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, - inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, + inv_cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, @@ -240,12 +251,11 @@ ::p3_main_internal_disp( p3_main_part2_disp( nj, nk, runtime_options.max_total_ni, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, inv_dt, - lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, + lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, lookup_tables.revap_table_vals, pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, - qv, th, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, - latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, + qv, th, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, @@ -286,7 +296,7 @@ ::p3_main_internal_disp( // homogeneous freezing f cloud and rain homogeneous_freezing_disp( - T_atm, inv_exner, latent_heat_fusion, nj, nk, ktop, kbot, kdir, qc, nc, qr, nr, qi, + T_atm, inv_exner, nj, nk, ktop, kbot, kdir, qc, nc, qr, nr, qi, ni, qm, bm, th, nucleationPossible, hydrometeorsPresent); // @@ -296,7 +306,7 @@ ::p3_main_internal_disp( p3_main_part3_disp( nj, nk_pack, runtime_options.max_total_ni, lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, rho, inv_rho, rhofaci, qv, th, qc, nc, qr, nr, qi, ni, - qm, bm, latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, + qm, bm, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr, nucleationPossible, hydrometeorsPresent, p3constants); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp index 3dae805255b0..28db7a6348dc 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp @@ -31,9 +31,6 @@ ::p3_main_part1_disp( const uview_2d& inv_cld_frac_l, const uview_2d& inv_cld_frac_i, const uview_2d& inv_cld_frac_r, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& T_atm, const uview_2d& rho, const uview_2d& inv_rho, @@ -73,17 +70,17 @@ ::p3_main_part1_disp( policy, KOKKOS_LAMBDA(const MemberType& team) { const Int i = team.league_rank(); - + p3_main_part1( team, nk, predictNc, prescribedCCN, dt, ekat::subview(pres, i), ekat::subview(dpres, i), ekat::subview(dz, i), ekat::subview(nc_nuceat_tend, i), ekat::subview(nccn_prescribed, i), ekat::subview(inv_exner, i), ekat::subview(exner, i), ekat::subview(inv_cld_frac_l, i), - ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), - ekat::subview(latent_heat_fusion, i), ekat::subview(T_atm, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), + ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), + ekat::subview(T_atm, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), ekat::subview(qv_sat_i, i), ekat::subview(qv_supersat_i, i), ekat::subview(rhofacr, i), ekat::subview(rhofaci, i), ekat::subview(acn, i), ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(qc_incld, i), ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), - ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), + ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), nucleationPossible(i), hydrometeorsPresent(i), p3constants); }); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp index 2b619d54bf33..91bb01989aef 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp @@ -10,6 +10,9 @@ namespace p3 { * this file, #include p3_functions.hpp instead. */ +#ifdef SCREAM_SYSTEM_WORKAROUND_P3_PART2 +#pragma clang optimize off +#endif template <> void Functions ::p3_main_part2_disp( @@ -59,9 +62,6 @@ ::p3_main_part2_disp( const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& qc_incld, const uview_2d& qr_incld, const uview_2d& qi_incld, @@ -104,25 +104,25 @@ ::p3_main_part2_disp( const Int i = team.league_rank(); if (!(nucleationPossible(i) || hydrometeorsPresent(i))) { - return; + return; } // ------------------------------------------------------------------------------------------ // main k-loop (for processes): p3_main_part2( team, nk_pack, max_total_ni, predictNc, do_prescribed_CCN, dt, inv_dt, - dnu_table_vals, ice_table_vals, collect_table_vals, revap_table_vals, + dnu_table_vals, ice_table_vals, collect_table_vals, revap_table_vals, ekat::subview(pres, i), ekat::subview(dpres, i), ekat::subview(dz, i), ekat::subview(nc_nuceat_tend, i), ekat::subview(inv_exner, i), - ekat::subview(exner, i), ekat::subview(inv_cld_frac_l, i), ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), - ekat::subview(ni_activated, i), ekat::subview(inv_qc_relvar, i), ekat::subview(cld_frac_i, i), ekat::subview(cld_frac_l, i), - ekat::subview(cld_frac_r, i), ekat::subview(qv_prev, i), ekat::subview(t_prev, i), ekat::subview(T_atm, i), ekat::subview(rho, i), - ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), ekat::subview(qv_sat_i, i), ekat::subview(qv_supersat_i, i), ekat::subview(rhofacr, i), - ekat::subview(rhofaci, i), ekat::subview(acn, i), ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), - ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), - ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), ekat::subview(latent_heat_fusion, i), ekat::subview(qc_incld, i), - ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), + ekat::subview(exner, i), ekat::subview(inv_cld_frac_l, i), ekat::subview(inv_cld_frac_i, i), ekat::subview(inv_cld_frac_r, i), + ekat::subview(ni_activated, i), ekat::subview(inv_qc_relvar, i), ekat::subview(cld_frac_i, i), ekat::subview(cld_frac_l, i), + ekat::subview(cld_frac_r, i), ekat::subview(qv_prev, i), ekat::subview(t_prev, i), ekat::subview(T_atm, i), ekat::subview(rho, i), + ekat::subview(inv_rho, i), ekat::subview(qv_sat_l, i), ekat::subview(qv_sat_i, i), ekat::subview(qv_supersat_i, i), ekat::subview(rhofacr, i), + ekat::subview(rhofaci, i), ekat::subview(acn, i), ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), + ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), + ekat::subview(qc_incld, i), + ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), ekat::subview(mu_c, i), ekat::subview(nu, i), ekat::subview(lamc, i), ekat::subview(cdist, i), - ekat::subview(cdist1, i), ekat::subview(cdistr, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(logn0r, i), + ekat::subview(cdist1, i), ekat::subview(cdistr, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(logn0r, i), ekat::subview(qv2qi_depos_tend, i), ekat::subview(precip_total_tend, i), ekat::subview(nevapr, i), ekat::subview(qr_evap_tend, i), ekat::subview(vap_liq_exchange, i), ekat::subview(vap_ice_exchange, i), ekat::subview(liq_ice_exchange, i), ekat::subview(pratot, i), ekat::subview(prctot, i), hydrometeorsPresent(i), nk, p3constants); @@ -130,7 +130,9 @@ ::p3_main_part2_disp( if (!hydrometeorsPresent(i)) return; }); } - +#ifdef SCREAM_SYSTEM_WORKAROUND_P3_PART2 +#pragma clang optimize on +#endif } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp index 6d591a7e9328..fd41fb5f0fe1 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp @@ -38,8 +38,6 @@ ::p3_main_part3_disp( const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, const uview_2d& mu_c, const uview_2d& nu, const uview_2d& lamc, @@ -77,11 +75,11 @@ ::p3_main_part3_disp( // p3_main_part3( team, nk_pack, max_total_ni, dnu_table_vals, ice_table_vals, ekat::subview(inv_exner, i), ekat::subview(cld_frac_l, i), ekat::subview(cld_frac_r, i), - ekat::subview(cld_frac_i, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(qv, i), - ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), - ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(latent_heat_vapor, i), ekat::subview(latent_heat_sublim, i), + ekat::subview(cld_frac_i, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofaci, i), ekat::subview(qv, i), + ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), + ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(mu_c, i), ekat::subview(nu, i), ekat::subview(lamc, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), - ekat::subview(vap_liq_exchange, i), ekat::subview(ze_rain, i), ekat::subview(ze_ice, i), ekat::subview(diag_vm_qi, i), ekat::subview(diag_eff_radius_qi, i), + ekat::subview(vap_liq_exchange, i), ekat::subview(ze_rain, i), ekat::subview(ze_ice, i), ekat::subview(diag_vm_qi, i), ekat::subview(diag_eff_radius_qi, i), ekat::subview(diag_diam_qi, i), ekat::subview(rho_qi, i), ekat::subview(diag_equiv_reflectivity, i), ekat::subview(diag_eff_radius_qc, i), ekat::subview(diag_eff_radius_qr, i), p3constants); diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index d886478ac2af..c414d7093f62 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -95,11 +95,13 @@ void P3Microphysics::set_grids(const std::shared_ptr grids_m add_field ("T_prev_micro_step", scalar3d_layout_mid, K, grid_name, ps); // Diagnostic Outputs: (all fields are just outputs w.r.t. P3) - add_field("precip_liq_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); - add_field("precip_ice_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); - add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name, ps); - add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name, ps); - add_field("eff_radius_qr", scalar3d_layout_mid, micron, grid_name, ps); + add_field("precip_liq_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); + add_field("precip_ice_surf_mass", scalar2d_layout, kg/m2, grid_name, "ACCUMULATED"); + add_field("eff_radius_qc", scalar3d_layout_mid, micron, grid_name, ps); + add_field("eff_radius_qi", scalar3d_layout_mid, micron, grid_name, ps); + add_field("eff_radius_qr", scalar3d_layout_mid, micron, grid_name, ps); + add_field("precip_total_tend", scalar3d_layout_mid, kg/(kg*s), grid_name, ps); + add_field("nevapr", scalar3d_layout_mid, kg/(kg*s), grid_name, ps); // History Only: (all fields are just outputs and are really only meant for I/O purposes) // TODO: These should be averaged over subcycle as well. But there is no simple mechanism @@ -150,10 +152,14 @@ void P3Microphysics::init_buffers(const ATMBufferManager &buffer_manager) Real* mem = reinterpret_cast(buffer_manager.get_memory()); // 1d scalar views - m_buffer.precip_liq_surf_flux = decltype(m_buffer.precip_liq_surf_flux)(mem, m_num_cols); - mem += m_buffer.precip_liq_surf_flux.size(); - m_buffer.precip_ice_surf_flux = decltype(m_buffer.precip_ice_surf_flux)(mem, m_num_cols); - mem += m_buffer.precip_ice_surf_flux.size(); + using scalar_1d_view_t = decltype(m_buffer.precip_liq_surf_flux); + scalar_1d_view_t* _1d_scalar_view_ptrs[Buffer::num_1d_scalar] = { + &m_buffer.precip_liq_surf_flux, &m_buffer.precip_ice_surf_flux + }; + for (int i=0; isize(); + } // 2d scalar views m_buffer.col_location = decltype(m_buffer.col_location)(mem, m_num_cols, 3); @@ -165,26 +171,38 @@ void P3Microphysics::init_buffers(const ATMBufferManager &buffer_manager) const Int nk_pack = ekat::npack(m_num_levs); const Int nk_pack_p1 = ekat::npack(m_num_levs+1); - m_buffer.inv_exner = decltype(m_buffer.inv_exner)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.inv_exner.size(); - m_buffer.th_atm = decltype(m_buffer.th_atm)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.th_atm.size(); - m_buffer.cld_frac_l = decltype(m_buffer.cld_frac_l)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.cld_frac_l.size(); - m_buffer.cld_frac_i = decltype(m_buffer.cld_frac_i)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.cld_frac_i.size(); - m_buffer.dz = decltype(m_buffer.dz)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.dz.size(); - m_buffer.qv2qi_depos_tend = decltype(m_buffer.qv2qi_depos_tend)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.qv2qi_depos_tend.size(); - m_buffer.rho_qi = decltype(m_buffer.rho_qi)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.rho_qi.size(); - m_buffer.precip_liq_flux = decltype(m_buffer.precip_liq_flux)(s_mem, m_num_cols, nk_pack_p1); - s_mem += m_buffer.precip_liq_flux.size(); - m_buffer.precip_ice_flux = decltype(m_buffer.precip_ice_flux)(s_mem, m_num_cols, nk_pack_p1); - s_mem += m_buffer.precip_ice_flux.size(); - m_buffer.unused = decltype(m_buffer.unused)(s_mem, m_num_cols, nk_pack); - s_mem += m_buffer.unused.size(); + using spack_2d_view_t = decltype(m_buffer.inv_exner); + spack_2d_view_t* _2d_spack_mid_view_ptrs[Buffer::num_2d_vector] = { + &m_buffer.inv_exner, &m_buffer.th_atm, &m_buffer.cld_frac_l, &m_buffer.cld_frac_i, + &m_buffer.dz, &m_buffer.qv2qi_depos_tend, &m_buffer.rho_qi, &m_buffer.unused +#ifdef SCREAM_P3_SMALL_KERNELS + , &m_buffer.mu_r, &m_buffer.T_atm, &m_buffer.lamr, &m_buffer.logn0r, &m_buffer.nu, + &m_buffer.cdist, &m_buffer.cdist1, &m_buffer.cdistr, &m_buffer.inv_cld_frac_i, + &m_buffer.inv_cld_frac_l, &m_buffer.inv_cld_frac_r, &m_buffer.qc_incld, &m_buffer.qr_incld, + &m_buffer.qi_incld, &m_buffer.qm_incld, &m_buffer.nc_incld, &m_buffer.nr_incld, + &m_buffer.ni_incld, &m_buffer.bm_incld, &m_buffer.inv_dz, &m_buffer.inv_rho, &m_buffer.ze_ice, + &m_buffer.ze_rain, &m_buffer.prec, &m_buffer.rho, &m_buffer.rhofacr, &m_buffer.rhofaci, + &m_buffer.acn, &m_buffer.qv_sat_l, &m_buffer.qv_sat_i, &m_buffer.sup, &m_buffer.qv_supersat_i, + &m_buffer.tmparr2, &m_buffer.exner, &m_buffer.diag_equiv_reflectivity, &m_buffer.diag_vm_qi, + &m_buffer.diag_diam_qi, &m_buffer.pratot, &m_buffer.prctot, &m_buffer.qtend_ignore, + &m_buffer.ntend_ignore, &m_buffer.mu_c, &m_buffer.lamc, &m_buffer.qr_evap_tend, &m_buffer.v_qc, + &m_buffer.v_nc, &m_buffer.flux_qx, &m_buffer.flux_nx, &m_buffer.v_qit, &m_buffer.v_nit, + &m_buffer.flux_nit, &m_buffer.flux_bir, &m_buffer.flux_qir, &m_buffer.flux_qit, &m_buffer.v_qr, + &m_buffer.v_nr +#endif + }; + for (int i=0; isize(); + } + + spack_2d_view_t* _2d_spack_int_view_ptrs[Buffer::num_2dp1_vector] = { + &m_buffer.precip_liq_flux, &m_buffer.precip_ice_flux + }; + for (int i=0; isize(); + } // WSM data m_buffer.wsm_data = s_mem; @@ -307,6 +325,8 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) diag_outputs.diag_eff_radius_qc = get_field_out("eff_radius_qc").get_view(); diag_outputs.diag_eff_radius_qi = get_field_out("eff_radius_qi").get_view(); diag_outputs.diag_eff_radius_qr = get_field_out("eff_radius_qr").get_view(); + diag_outputs.precip_total_tend = get_field_out("precip_total_tend").get_view(); + diag_outputs.nevapr = get_field_out("nevapr").get_view(); diag_outputs.precip_liq_surf = m_buffer.precip_liq_surf_flux; diag_outputs.precip_ice_surf = m_buffer.precip_ice_surf_flux; @@ -320,6 +340,66 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) history_only.liq_ice_exchange = get_field_out("micro_liq_ice_exchange").get_view(); history_only.vap_liq_exchange = get_field_out("micro_vap_liq_exchange").get_view(); history_only.vap_ice_exchange = get_field_out("micro_vap_ice_exchange").get_view(); +#ifdef SCREAM_P3_SMALL_KERNELS + // Temporaries + temporaries.mu_r = m_buffer.mu_r; + temporaries.T_atm = m_buffer.T_atm; + temporaries.lamr = m_buffer.lamr; + temporaries.logn0r = m_buffer.logn0r; + temporaries.nu = m_buffer.nu; + temporaries.cdist = m_buffer.cdist; + temporaries.cdist1 = m_buffer.cdist1; + temporaries.cdistr = m_buffer.cdistr; + temporaries.inv_cld_frac_i = m_buffer.inv_cld_frac_i; + temporaries.inv_cld_frac_l = m_buffer.inv_cld_frac_l; + temporaries.inv_cld_frac_r = m_buffer.inv_cld_frac_r; + temporaries.qc_incld = m_buffer.qc_incld; + temporaries.qr_incld = m_buffer.qr_incld; + temporaries.qi_incld = m_buffer.qi_incld; + temporaries.qm_incld = m_buffer.qm_incld; + temporaries.nc_incld = m_buffer.nc_incld; + temporaries.nr_incld = m_buffer.nr_incld; + temporaries.ni_incld = m_buffer.ni_incld; + temporaries.bm_incld = m_buffer.bm_incld; + temporaries.inv_dz = m_buffer.inv_dz; + temporaries.inv_rho = m_buffer.inv_rho; + temporaries.ze_ice = m_buffer.ze_ice; + temporaries.ze_rain = m_buffer.ze_rain; + temporaries.prec = m_buffer.prec; + temporaries.rho = m_buffer.rho; + temporaries.rhofacr = m_buffer.rhofacr; + temporaries.rhofaci = m_buffer.rhofaci; + temporaries.acn = m_buffer.acn; + temporaries.qv_sat_l = m_buffer.qv_sat_l; + temporaries.qv_sat_i = m_buffer.qv_sat_i; + temporaries.sup = m_buffer.sup; + temporaries.qv_supersat_i = m_buffer.qv_supersat_i; + temporaries.tmparr2 = m_buffer.tmparr2; + temporaries.exner = m_buffer.exner; + temporaries.diag_equiv_reflectivity = m_buffer.diag_equiv_reflectivity; + temporaries.diag_vm_qi = m_buffer.diag_vm_qi; + temporaries.diag_diam_qi = m_buffer.diag_diam_qi; + temporaries.pratot = m_buffer.pratot; + temporaries.prctot = m_buffer.prctot; + temporaries.qtend_ignore = m_buffer.qtend_ignore; + temporaries.ntend_ignore = m_buffer.ntend_ignore; + temporaries.mu_c = m_buffer.mu_c; + temporaries.lamc = m_buffer.lamc; + temporaries.qr_evap_tend = m_buffer.qr_evap_tend; + temporaries.v_qc = m_buffer.v_qc; + temporaries.v_nc = m_buffer.v_nc; + temporaries.flux_qx = m_buffer.flux_qx; + temporaries.flux_nx = m_buffer.flux_nx; + temporaries.v_qit = m_buffer.v_qit; + temporaries.v_nit = m_buffer.v_nit; + temporaries.flux_nit = m_buffer.flux_nit; + temporaries.flux_bir = m_buffer.flux_bir; + temporaries.flux_qir = m_buffer.flux_qir; + temporaries.flux_qit = m_buffer.flux_qit; + temporaries.v_qr = m_buffer.v_qr; + temporaries.v_nr = m_buffer.v_nr; +#endif + // -- Set values for the post-amble structure p3_postproc.set_variables(m_num_cols,nk_pack, prog_state.th,pmid,pmid_dry,T_atm,t_prev, diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp index 47d543ded022..32e3b76d1a0d 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp @@ -158,7 +158,7 @@ class P3Microphysics : public AtmosphereProcess // Assigning local variables void set_variables(const int ncol, const int npack, const view_2d_const& pmid_, const view_2d_const& pmid_dry_, - const view_2d_const& pseudo_density_, + const view_2d_const& pseudo_density_, const view_2d_const& pseudo_density_dry_, const view_2d& T_atm_, const view_2d_const& cld_frac_t_, const view_2d& qv_, const view_2d& qc_, const view_2d& nc_, const view_2d& qr_, const view_2d& nr_, const view_2d& qi_, @@ -361,7 +361,11 @@ class P3Microphysics : public AtmosphereProcess // 1d view scalar, size (ncol) static constexpr int num_1d_scalar = 2; //no 2d vars now, but keeping 1d struct for future expansion // 2d view packed, size (ncol, nlev_packs) +#ifdef SCREAM_P3_SMALL_KERNELS + static constexpr int num_2d_vector = 64; +#else static constexpr int num_2d_vector = 8; +#endif static constexpr int num_2dp1_vector = 2; uview_1d precip_liq_surf_flux; @@ -377,6 +381,21 @@ class P3Microphysics : public AtmosphereProcess uview_2d precip_ice_flux; //nlev+1 uview_2d unused; +#ifdef SCREAM_P3_SMALL_KERNELS + uview_2d + mu_r, T_atm, lamr, logn0r, nu, cdist, cdist1, cdistr, + inv_cld_frac_i, inv_cld_frac_l, inv_cld_frac_r, + qc_incld, qr_incld, qi_incld, qm_incld, + nc_incld, nr_incld, ni_incld, bm_incld, + inv_dz, inv_rho, ze_ice, ze_rain, prec, rho, rhofacr, + rhofaci, acn, qv_sat_l, qv_sat_i, sup, qv_supersat_i, + tmparr2, exner, diag_equiv_reflectivity, diag_vm_qi, + diag_diam_qi, pratot, prctot, qtend_ignore, ntend_ignore, + mu_c, lamc, qr_evap_tend, v_qc, v_nc, flux_qx, flux_nx, + v_qit, v_nit, flux_nit, flux_bir, flux_qir, flux_qit, + v_qr, v_nr; +#endif + suview_2d col_location; Spack* wsm_data; @@ -410,6 +429,9 @@ class P3Microphysics : public AtmosphereProcess P3F::P3DiagnosticOutputs diag_outputs; P3F::P3HistoryOnly history_only; P3F::P3LookupTables lookup_tables; +#ifdef SCREAM_P3_SMALL_KERNELS + P3F::P3Temporaries temporaries; +#endif P3F::P3Infrastructure infrastructure; P3F::P3Runtime runtime_options; p3_preamble p3_preproc; diff --git a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp index 9ccfdaf20220..cdb156d71f62 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp @@ -29,7 +29,11 @@ void P3Microphysics::run_impl (const double dt) get_field_out("micro_vap_ice_exchange").deep_copy(0.0); P3F::p3_main(runtime_options, prog_state, diag_inputs, diag_outputs, infrastructure, - history_only, lookup_tables, workspace_mgr, m_num_cols, m_num_levs, m_p3constants); + history_only, lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + temporaries, +#endif + workspace_mgr, m_num_cols, m_num_levs, m_p3constants); // Conduct the post-processing of the p3_main output. Kokkos::parallel_for( diff --git a/components/eamxx/src/physics/p3/eti/p3_get_latent_heat.cpp b/components/eamxx/src/physics/p3/eti/p3_get_latent_heat.cpp deleted file mode 100644 index 87e41c934587..000000000000 --- a/components/eamxx/src/physics/p3/eti/p3_get_latent_heat.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "p3_get_latent_heat_impl.hpp" - -namespace scream { -namespace p3 { - -/* - * Explicit instantiation for doing conservation functions on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace p3 -} // namespace scream diff --git a/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp index 5455bfaa4c14..5560fcfe9123 100644 --- a/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_evaporate_rain_impl.hpp @@ -69,7 +69,7 @@ ::evaporate_rain( const Spack& cld_frac_l, const Spack& cld_frac_r, const Spack& qv, const Spack& qv_prev, const Spack& qv_sat_l, const Spack& qv_sat_i, const Spack& ab, const Spack& abi, const Spack& epsr, const Spack& epsi_tot, const Spack& t_atm, const Spack& t_atm_prev, - const Spack& latent_heat_sublim, const Spack& dqsdt, const Scalar& dt, + const Spack& dqsdt, const Scalar& dt, Spack& qr2qv_evap_tend, Spack& nr_evap_tend, const Smask& context) { @@ -92,6 +92,8 @@ ::evaporate_rain( constexpr Scalar QSMALL = C::QSMALL; constexpr Scalar Tmelt = C::Tmelt; constexpr Scalar inv_cp = 1/C::Cpair; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //Compute absolute supersaturation. //Ignore the difference between clear-sky and cell-ave qv and T @@ -133,9 +135,9 @@ ::evaporate_rain( const Smask not_freezing = !is_freezing && context; Spack eps_eff, A_c; if (is_freezing.any()){ - eps_eff.set(is_freezing,epsr + epsi_tot*(1 + latent_heat_sublim*inv_cp*dqsdt)/abi); + eps_eff.set(is_freezing,epsr + epsi_tot*(1 + (latvap+latice)*inv_cp*dqsdt)/abi); A_c.set(is_freezing,(qv - qv_prev)*inv_dt - dqsdt*(t_atm-t_atm_prev)*inv_dt - - (qv_sat_l - qv_sat_i)*(1 + latent_heat_sublim*inv_cp*dqsdt)/abi*epsi_tot ); + - (qv_sat_l - qv_sat_i)*(1 + (latvap+latice)*inv_cp*dqsdt)/abi*epsi_tot ); } if (not_freezing.any()){ eps_eff.set(not_freezing,epsr); diff --git a/components/eamxx/src/physics/p3/impl/p3_get_latent_heat_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_get_latent_heat_impl.hpp deleted file mode 100644 index 439a1e2c84d6..000000000000 --- a/components/eamxx/src/physics/p3/impl/p3_get_latent_heat_impl.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef P3_GET_LATENT_HEAT_IMPL_HPP -#define P3_GET_LATENT_HEAT_IMPL_HPP - -#include "p3_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace p3 { - -template -void Functions -::get_latent_heat(const Int& nj, const Int& nk, view_2d& v, view_2d& s, view_2d& f) -{ - constexpr Scalar latvap = C::LatVap; - constexpr Scalar latice = C::LatIce; - - Kokkos::deep_copy(v, latvap); - Kokkos::deep_copy(s, latvap + latice); - Kokkos::deep_copy(f, latice); -} - -} // namespace p3 -} // namespace scream - -#endif // P3_GET_LATENT_HEAT_IMPL_HPP diff --git a/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp index f46c4aaebb5f..ea42d7995ba1 100644 --- a/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_get_time_space_phys_variables_impl.hpp @@ -11,7 +11,7 @@ template KOKKOS_FUNCTION void Functions ::get_time_space_phys_variables( - const Spack& T_atm, const Spack& pres, const Spack& rho, const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, + const Spack& T_atm, const Spack& pres, const Spack& rho, const Spack& qv_sat_l, const Spack& qv_sat_i, Spack& mu, Spack& dv, Spack& sc, Spack& dqsdt, Spack& dqsidt, Spack& ab, Spack& abi, Spack& kap, Spack& eii, const Smask& context) @@ -23,15 +23,17 @@ ::get_time_space_phys_variables( constexpr Scalar RV = C::RV; constexpr Scalar INV_CP = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; constexpr Scalar tval1 = 253.15; constexpr Scalar tval2 = 273.15; constexpr Scalar dtval = 20; //this is tval2-tval1, but specifying here as int to be BFB with F90. const auto dum = 1/(RV*square(T_atm)); - dqsdt.set(context, latent_heat_vapor*qv_sat_l*dum); - dqsidt.set(context, latent_heat_sublim*qv_sat_i*dum); - ab.set(context, 1+dqsdt*latent_heat_vapor*INV_CP); - abi.set(context, 1+dqsidt*latent_heat_sublim*INV_CP); + dqsdt.set(context, latvap*qv_sat_l*dum); + dqsidt.set(context, (latvap+latice)*qv_sat_i*dum); + ab.set(context, 1+dqsdt*latvap*INV_CP); + abi.set(context, 1+dqsidt*(latvap+latice)*INV_CP); kap.set(context, sp(1.414e+3)*mu); //very simple temperature dependent aggregation efficiency diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp index cd8d9ce7ef0e..319d10f64ef7 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_cldliq_wet_growth_impl.hpp @@ -13,7 +13,7 @@ KOKKOS_FUNCTION void Functions ::ice_cldliq_wet_growth( const Spack& rho, const Spack& temp, const Spack& pres, const Spack& rhofaci, const Spack& table_val_qi2qr_melting, - const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, const Spack& dv, + const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& kap, const Spack& mu, const Spack& sc, const Spack& qv, const Spack& qc_incld, const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Smask& log_wetgrowth, Spack& qr2qi_collect_tend, Spack& qc2qi_collect_tend, Spack& qc_growth_rate, Spack& nr_ice_shed_tend, Spack& qc2qr_ice_shed_tend, const Smask& context) @@ -26,6 +26,8 @@ ::ice_cldliq_wet_growth( constexpr Scalar zero = C::ZERO; constexpr Scalar one = C::ONE; constexpr Scalar cpw = C::CpLiq; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; const auto t_is_negative = temp < tmelt; const auto qi_incld_ge_small = qi_incld >= qsmall; @@ -46,8 +48,8 @@ ::ice_cldliq_wet_growth( qc_growth_rate.set(any_if, ((table_val_qi2qr_melting+table_val_qi2qr_vent_melt*cbrt(sc)*sqrt(rhofaci*rho/mu))* - twopi*(rho*latent_heat_vapor*dv*(qsat0-qv)-(temp-tmelt)*kap)/ - (latent_heat_fusion+cpw*(temp-tmelt)))*ni_incld); + twopi*(rho*latvap*dv*(qsat0-qv)-(temp-tmelt)*kap)/ + (latice+cpw*(temp-tmelt)))*ni_incld); qc_growth_rate.set(any_if, max(qc_growth_rate, zero)); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp index dc6c8d16256f..a4b86806ad0c 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_melting_impl.hpp @@ -13,7 +13,7 @@ KOKKOS_FUNCTION void Functions ::ice_melting( const Spack& rho, const Spack& T_atm, const Spack& pres, const Spack& rhofaci, - const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, + const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& sc, const Spack& mu, const Spack& kap, const Spack& qv, const Spack& qi_incld, const Spack& ni_incld, Spack& qi2qr_melt_tend, Spack& ni2nr_melt_tend, const Smask& context) @@ -29,6 +29,8 @@ ::ice_melting( const auto Pi = C::Pi; const auto QSMALL = C::QSMALL; const auto Tmelt = C::Tmelt; + const auto latvap = C::LatVap; + const auto latice = C::LatIce; //Find cells above freezing AND which have ice const auto has_melt_qi = (qi_incld >= QSMALL ) && (T_atm > Tmelt) && context; @@ -38,8 +40,8 @@ ::ice_melting( const auto qsat0 = physics::qv_sat_dry(Spack(Tmelt), pres, false, context, physics::MurphyKoop, "p3::ice_melting"); //"false" here means NOT saturation w/ respect to ice. qi2qr_melt_tend.set(has_melt_qi, ( (table_val_qi2qr_melting+table_val_qi2qr_vent_melt*cbrt(sc)*sqrt(rhofaci*rho/mu)) - *((T_atm-Tmelt)*kap-rho*latent_heat_vapor*dv*(qsat0-qv)) - * 2 * Pi /latent_heat_fusion)*ni_incld ); + *((T_atm-Tmelt)*kap-rho*latvap*dv*(qsat0-qv)) + * 2 * Pi /latice)*ni_incld ); //make sure qi2qr_melt_tend is always negative qi2qr_melt_tend = max(qi2qr_melt_tend, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp index 29232c69e68f..1273d7bbc206 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp @@ -210,7 +210,6 @@ void Functions ::homogeneous_freezing( const uview_1d& T_atm, const uview_1d& inv_exner, - const uview_1d& latent_heat_fusion, const MemberType& team, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_1d& qc, @@ -228,6 +227,7 @@ ::homogeneous_freezing( constexpr Scalar T_homogfrz = C::T_homogfrz; constexpr Scalar inv_rho_rimeMax = C::INV_RHO_RIMEMAX; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latice = C::LatIce; const Int kmin_scalar = ( kdir == 1 ? kbot : ktop); const Int kmax_scalar = ( kdir == 1 ? ktop : kbot); @@ -254,13 +254,13 @@ ::homogeneous_freezing( qi(pk).set(qc_ge_small, qi(pk) + Qc_nuc); bm(pk).set(qc_ge_small, bm(pk) + Qc_nuc*inv_rho_rimeMax); ni(pk).set(qc_ge_small, ni(pk) + Nc_nuc); - th_atm(pk).set (qc_ge_small, th_atm(pk) + inv_exner(pk)*Qc_nuc*latent_heat_fusion(pk)*inv_cp); + th_atm(pk).set (qc_ge_small, th_atm(pk) + inv_exner(pk)*Qc_nuc*latice*inv_cp); qm(pk).set(qr_ge_small, qm(pk) + Qr_nuc); qi(pk).set(qr_ge_small, qi(pk) + Qr_nuc); bm(pk).set(qr_ge_small, bm(pk) + Qr_nuc*inv_rho_rimeMax); ni(pk).set(qr_ge_small, ni(pk) + Nr_nuc); - th_atm(pk).set (qr_ge_small, th_atm(pk) + inv_exner(pk)*Qr_nuc*latent_heat_fusion(pk)*inv_cp); + th_atm(pk).set (qr_ge_small, th_atm(pk) + inv_exner(pk)*Qr_nuc*latice*inv_cp); qc(pk).set(qc_ge_small, 0); nc(pk).set(qc_ge_small, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp index 1bed57bad630..091aa20f58d4 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_supersat_conservation_impl.hpp @@ -13,11 +13,14 @@ namespace p3 { template KOKKOS_FUNCTION -void Functions::ice_supersat_conservation(Spack& qv2qi_vapdep_tend, Spack& qv2qi_nucleat_tend, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& latent_heat_sublim, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context) +void Functions::ice_supersat_conservation(Spack& qv2qi_vapdep_tend, Spack& qv2qi_nucleat_tend, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar cp = C::CP; constexpr Scalar rv = C::RH2O; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + constexpr Scalar latsublim2 = (latvap+latice)*(latvap+latice); const auto qv_sink = qv2qi_vapdep_tend + qv2qi_nucleat_tend; // in [kg/kg] cell-avg values @@ -25,7 +28,7 @@ void Functions::ice_supersat_conservation(Spack& qv2qi_vapdep_tend, Spack& if (mask.any()) { // --- Available water vapor for deposition/nucleation auto qv_avail = (qv + (qi2qv_sublim_tend+qr2qv_evap_tend)*dt - qv_sat_i) / - (1 + square(latent_heat_sublim)*qv_sat_i / (cp*rv*square(t_atm)) ) / dt; + (1 + latsublim2*qv_sat_i / (cp*rv*square(t_atm)) ) / dt; // --- Only excess water vapor can be limited qv_avail = max(qv_avail, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index b05b713aec29..964ec21be700 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -87,13 +87,11 @@ ::p3_main_internal( const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; - - view_2d latent_heat_sublim("latent_heat_sublim", nj, nk), latent_heat_vapor("latent_heat_vapor", nj, nk), latent_heat_fusion("latent_heat_fusion", nj, nk); - - get_latent_heat(nj, nk, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion); + using ScratchViewType = Kokkos::View; const Int nk_pack = ekat::npack(nk); - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + const auto scratch_size = ScratchViewType::shmem_size(2); + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack).set_scratch_size(0, Kokkos::PerTeam(scratch_size)); // load constants into local vars const Scalar inv_dt = 1 / infrastructure.dt; @@ -102,9 +100,6 @@ ::p3_main_internal( const Int kbot = kdir == -1 ? nk-1 : 0; constexpr bool debug_ABORT = false; - // per-column bools - view_2d bools("bools", nj, 2); - // we do not want to measure init stuff auto start = std::chrono::steady_clock::now(); @@ -142,9 +137,9 @@ ::p3_main_internal( qtend_ignore, ntend_ignore, // Variables still used in F90 but removed from C++ interface - mu_c, lamc, precip_total_tend, nevapr, qr_evap_tend; + mu_c, lamc, qr_evap_tend; - workspace.template take_many_and_reset<46>( + workspace.template take_many_and_reset<44>( { "mu_r", "T_atm", "lamr", "logn0r", "nu", "cdist", "cdist1", "cdistr", "inv_cld_frac_i", "inv_cld_frac_l", "inv_cld_frac_r", "qc_incld", "qr_incld", "qi_incld", "qm_incld", @@ -153,7 +148,7 @@ ::p3_main_internal( "rhofacr", "rhofaci", "acn", "qv_sat_l", "qv_sat_i", "sup", "qv_supersat_i", "tmparr1", "exner", "diag_equiv_reflectivity", "diag_vm_qi", "diag_diam_qi", "pratot", "prctot", "qtend_ignore", "ntend_ignore", - "mu_c", "lamc", "precip_total_tend", "nevapr", "qr_evap_tend" + "mu_c", "lamc", "qr_evap_tend" }, { &mu_r, &T_atm, &lamr, &logn0r, &nu, &cdist, &cdist1, &cdistr, @@ -162,10 +157,10 @@ ::p3_main_internal( &inv_dz, &inv_rho, &ze_ice, &ze_rain, &prec, &rho, &rhofacr, &rhofaci, &acn, &qv_sat_l, &qv_sat_i, &sup, &qv_supersat_i, &tmparr1, &exner, &diag_equiv_reflectivity, &diag_vm_qi, &diag_diam_qi, - &pratot, &prctot, &qtend_ignore, &ntend_ignore, - &mu_c, &lamc, &precip_total_tend, &nevapr, &qr_evap_tend + &pratot, &prctot, &qtend_ignore, &ntend_ignore, + &mu_c, &lamc, &qr_evap_tend }); - + // Get single-column subviews of all inputs, shouldn't need any i-indexing // after this. const auto opres = ekat::subview(diagnostic_inputs.pres, i); @@ -197,18 +192,19 @@ ::p3_main_internal( const auto orho_qi = ekat::subview(diagnostic_outputs.rho_qi, i); const auto oprecip_liq_flux = ekat::subview(diagnostic_outputs.precip_liq_flux, i); const auto oprecip_ice_flux = ekat::subview(diagnostic_outputs.precip_ice_flux, i); + const auto oprecip_total_tend = ekat::subview(diagnostic_outputs.precip_total_tend, i); + const auto onevapr = ekat::subview(diagnostic_outputs.nevapr, i); const auto oliq_ice_exchange = ekat::subview(history_only.liq_ice_exchange, i); const auto ovap_liq_exchange = ekat::subview(history_only.vap_liq_exchange, i); const auto ovap_ice_exchange = ekat::subview(history_only.vap_ice_exchange, i); - const auto olatent_heat_vapor = ekat::subview(latent_heat_vapor, i); - const auto olatent_heat_sublim = ekat::subview(latent_heat_sublim, i); - const auto olatent_heat_fusion = ekat::subview(latent_heat_fusion, i); const auto oqv_prev = ekat::subview(diagnostic_inputs.qv_prev, i); const auto ot_prev = ekat::subview(diagnostic_inputs.t_prev, i); - // Need to watch out for race conditions with these shared variables - bool &nucleationPossible = bools(i, 0); - bool &hydrometeorsPresent = bools(i, 1); + // Use Kokkos' scratch pad for allocating 2 bools + // per team to determine early exits + ScratchViewType bools(team.team_scratch(0), 2); + bool &nucleationPossible = bools(0); + bool &hydrometeorsPresent = bools(1); view_1d_ptr_array zero_init = { &mu_r, &lamr, &logn0r, &nu, &cdist, &cdist1, &cdistr, @@ -216,7 +212,7 @@ ::p3_main_internal( &nc_incld, &nr_incld, &ni_incld, &bm_incld, &inv_rho, &prec, &rho, &rhofacr, &rhofaci, &acn, &qv_sat_l, &qv_sat_i, &sup, &qv_supersat_i, &tmparr1, &qtend_ignore, &ntend_ignore, - &mu_c, &lamc, &orho_qi, &oqv2qi_depos_tend, &precip_total_tend, &nevapr, &oprecip_liq_flux, &oprecip_ice_flux + &mu_c, &lamc, &orho_qi, &oqv2qi_depos_tend, &oprecip_total_tend, &onevapr, &oprecip_liq_flux, &oprecip_ice_flux }; // initialize @@ -230,7 +226,7 @@ ::p3_main_internal( p3_main_part1( team, nk, infrastructure.predictNc, infrastructure.prescribedCCN, infrastructure.dt, opres, odpres, odz, onc_nuceat_tend, onccn_prescribed, oinv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, - inv_cld_frac_r, olatent_heat_vapor, olatent_heat_sublim, olatent_heat_fusion, + inv_cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, @@ -249,10 +245,10 @@ ::p3_main_internal( lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, lookup_tables.collect_table_vals, lookup_tables.revap_table_vals, opres, odpres, odz, onc_nuceat_tend, oinv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, oni_activated, oinv_qc_relvar, ocld_frac_i, ocld_frac_l, ocld_frac_r, oqv_prev, ot_prev, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, - oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, olatent_heat_vapor, - olatent_heat_sublim, olatent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, + oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, + qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, - mu_r, lamr, logn0r, oqv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, + mu_r, lamr, logn0r, oqv2qi_depos_tend, oprecip_total_tend, onevapr, qr_evap_tend, ovap_liq_exchange, ovap_ice_exchange, oliq_ice_exchange, pratot, prctot, hydrometeorsPresent, nk, p3constants); @@ -293,7 +289,7 @@ ::p3_main_internal( // homogeneous freezing of cloud and rain homogeneous_freezing( - T_atm, oinv_exner, olatent_heat_fusion, team, nk, ktop, kbot, kdir, oqc, onc, oqr, onr, oqi, + T_atm, oinv_exner, team, nk, ktop, kbot, kdir, oqc, onc, oqr, onr, oqi, oni, oqm, obm, oth); // @@ -303,7 +299,7 @@ ::p3_main_internal( p3_main_part3( team, nk_pack, runtime_options.max_total_ni, lookup_tables.dnu_table_vals, lookup_tables.ice_table_vals, oinv_exner, ocld_frac_l, ocld_frac_r, ocld_frac_i, rho, inv_rho, rhofaci, oqv, oth, oqc, onc, oqr, onr, oqi, oni, - oqm, obm, olatent_heat_vapor, olatent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, + oqm, obm, mu_c, nu, lamc, mu_r, lamr, ovap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, odiag_eff_radius_qi, diag_diam_qi, orho_qi, diag_equiv_reflectivity, odiag_eff_radius_qc, odiag_eff_radius_qr, p3constants); @@ -342,22 +338,15 @@ ::p3_main( const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + const P3Temporaries& temporaries, +#endif const WorkspaceManager& workspace_mgr, Int nj, Int nk, const physics::P3_Constants & p3constants) { -#ifndef SCREAM_SMALL_KERNELS - return p3_main_internal(runtime_options, - prognostic_state, - diagnostic_inputs, - diagnostic_outputs, - infrastructure, - history_only, - lookup_tables, - workspace_mgr, - nj, nk, p3constants); -#else +#ifdef SCREAM_P3_SMALL_KERNELS return p3_main_internal_disp(runtime_options, prognostic_state, diagnostic_inputs, @@ -365,8 +354,19 @@ ::p3_main( infrastructure, history_only, lookup_tables, + temporaries, workspace_mgr, nj, nk, p3constants); +#else + return p3_main_internal(runtime_options, + prognostic_state, + diagnostic_inputs, + diagnostic_outputs, + infrastructure, + history_only, + lookup_tables, + workspace_mgr, + nj, nk, p3constants); #endif } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp index 439b0544adf9..6771a48d4c68 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp @@ -34,9 +34,6 @@ ::p3_main_part1( const uview_1d& inv_cld_frac_l, const uview_1d& inv_cld_frac_i, const uview_1d& inv_cld_frac_r, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& T_atm, const uview_1d& rho, const uview_1d& inv_rho, @@ -80,6 +77,8 @@ ::p3_main_part1( constexpr Scalar T_zerodegc = C::T_zerodegc; constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; const Scalar p3_spa_to_nc = p3constants.p3_spa_to_nc; @@ -124,7 +123,7 @@ ::p3_main_part1( auto drymass = qc(k) < qsmall; auto not_drymass = !drymass && range_mask; qv(k).set(drymass, qv(k) + qc(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qc(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qc(k) * latvap * inv_cp); qc(k).set(drymass, 0); nc(k).set(drymass, 0); if ( not_drymass.any() ) { @@ -147,7 +146,7 @@ ::p3_main_part1( drymass = qr(k) < qsmall; not_drymass = !drymass && range_mask; qv(k).set(drymass, qv(k) + qr(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qr(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qr(k) * latvap * inv_cp); qr(k).set(drymass, 0); nr(k).set(drymass, 0); if ( not_drymass.any() ) { @@ -157,7 +156,7 @@ ::p3_main_part1( drymass = (qi(k) < qsmall || (qi(k) < 1.e-8 && qv_supersat_i(k) < -0.1)); not_drymass = !drymass && range_mask; qv(k).set(drymass, qv(k) + qi(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * latent_heat_sublim(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * (latvap+latice) * inv_cp); qi(k).set(drymass, 0); ni(k).set(drymass, 0); qm(k).set(drymass, 0); @@ -168,7 +167,7 @@ ::p3_main_part1( drymass = (qi(k) >= qsmall && qi(k) < 1.e-8 && T_atm(k) >= T_zerodegc); qr(k).set(drymass, qr(k) + qi(k)); - th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * latent_heat_fusion(k) * inv_cp); + th_atm(k).set(drymass, th_atm(k) - inv_exner(k) * qi(k) * latice * inv_cp); qi(k).set(drymass, 0); ni(k).set(drymass, 0); qm(k).set(drymass, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp index c32fb7202ebb..d19dc579b7e9 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp @@ -65,9 +65,6 @@ ::p3_main_part2( const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& qc_incld, const uview_1d& qr_incld, const uview_1d& qi_incld, @@ -104,6 +101,8 @@ ::p3_main_part2( constexpr Scalar f2r = C::f2r; constexpr Scalar nmltratio = C::nmltratio; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; team.team_barrier(); hydrometeorsPresent = false; @@ -115,12 +114,12 @@ ::p3_main_part2( //compute mask to identify padded values in packs, which shouldn't be used in calculations const auto range_pack = ekat::range(k*Spack::n); const auto range_mask = range_pack < nk; - + // if relatively dry and no hydrometeors at this level, skip to end of k-loop (i.e. skip this level) const auto skip_all = ( !range_mask || (qc(k)= qsmall && not_skip_all; qv(k).set(qc_small, qv(k) + qc(k)); - th_atm(k).set(qc_small, th_atm(k) - inv_exner(k) * qc(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(qc_small, th_atm(k) - inv_exner(k) * qc(k) * latvap * inv_cp); qc(k).set(qc_small, 0); nc(k).set(qc_small, 0); @@ -464,7 +463,7 @@ ::p3_main_part2( } qv(k).set(qr_small, qv(k) + qr(k)); - th_atm(k).set(qr_small, th_atm(k) - inv_exner(k) * qr(k) * latent_heat_vapor(k) * inv_cp); + th_atm(k).set(qr_small, th_atm(k) - inv_exner(k) * qr(k) * latvap * inv_cp); qr(k).set(qr_small, 0); nr(k).set(qr_small, 0); @@ -473,7 +472,7 @@ ::p3_main_part2( } qv(k).set(qi_small, qv(k) + qi(k)); - th_atm(k).set(qi_small, th_atm(k) - inv_exner(k) * qi(k) * latent_heat_sublim(k) * inv_cp); + th_atm(k).set(qi_small, th_atm(k) - inv_exner(k) * qi(k) * (latvap+latice) * inv_cp); qi(k).set(qi_small, 0); ni(k).set(qi_small, 0); qm(k).set(qi_small, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp index 441bcb9feb9b..febb00ef4d31 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp @@ -41,8 +41,6 @@ ::p3_main_part3( const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, const uview_1d& mu_c, const uview_1d& nu, const uview_1d& lamc, @@ -63,6 +61,8 @@ ::p3_main_part3( constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; constexpr Scalar nsmall = C::NSMALL; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; Kokkos::parallel_for( Kokkos::TeamVectorRange(team, nk_pack), [&] (Int k) { @@ -93,7 +93,7 @@ ::p3_main_part3( } if (qc_small.any()) { qv(k) .set(qc_small, qv(k)+qc(k)); - th_atm(k) .set(qc_small, th_atm(k)-inv_exner(k)*qc(k)*latent_heat_vapor(k)*inv_cp); + th_atm(k) .set(qc_small, th_atm(k)-inv_exner(k)*qc(k)*latvap*inv_cp); vap_liq_exchange(k) .set(qc_small, vap_liq_exchange(k) - qc(k)); qc(k) .set(qc_small, 0); nc(k) .set(qc_small, 0); @@ -123,7 +123,7 @@ ::p3_main_part3( if (qr_small.any()) { qv(k) .set(qr_small, qv(k) + qr(k)); - th_atm(k) .set(qr_small, th_atm(k) - inv_exner(k)*qr(k)*latent_heat_vapor(k)*inv_cp); + th_atm(k) .set(qr_small, th_atm(k) - inv_exner(k)*qr(k)*latvap*inv_cp); vap_liq_exchange(k).set(qr_small, vap_liq_exchange(k) - qr(k)); qr(k) .set(qr_small, 0); nr(k) .set(qr_small, 0); @@ -185,7 +185,7 @@ ::p3_main_part3( ze_ice(k).set(qi_gt_small, ze_ice(k)*cld_frac_i(k)); qv(k).set(qi_small, qv(k) + qi(k)); - th_atm(k).set(qi_small, th_atm(k) - inv_exner(k)*qi(k)*latent_heat_sublim(k)*inv_cp); + th_atm(k).set(qi_small, th_atm(k) - inv_exner(k)*qi(k)*(latvap+latice)*inv_cp); qi(k).set(qi_small, 0); ni(k).set(qi_small, 0); qm(k).set(qi_small, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp index a9f265d8dcdd..10f728f1a079 100644 --- a/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_prevent_liq_supersaturation_impl.hpp @@ -14,7 +14,7 @@ namespace p3 { template KOKKOS_FUNCTION -void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, const Scalar& dt, const Spack& qv2qi_vapdep_tend, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context) +void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Scalar& dt, const Spack& qv2qi_vapdep_tend, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context) // Note: context masks cells which are just padding for packs or which don't have any condensate worth // performing calculations on. { @@ -23,6 +23,8 @@ void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& constexpr Scalar inv_cp = C::INV_CP; constexpr Scalar rv = C::RV; constexpr Scalar qsmall = C::QSMALL; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; Spack qv_sinks, qv_sources, qv_endstep, T_endstep, A, frac; @@ -37,8 +39,8 @@ void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& //Actual qv and T after microphys step qv_endstep.set(has_sources,qv - qv_sinks*dt + qv_sources*dt); - T_endstep.set(has_sources,t_atm + ( (qv_sinks-qi2qv_sublim_tend)*latent_heat_sublim*inv_cp - - qr2qv_evap_tend*latent_heat_vapor*inv_cp )*dt); + T_endstep.set(has_sources,t_atm + ( (qv_sinks-qi2qv_sublim_tend)*(latvap+latice)*inv_cp + - qr2qv_evap_tend*latvap*inv_cp )*dt); //qv we would have at end of step if we were saturated with respect to liquid const auto qsl = physics::qv_sat_dry(T_endstep,pres,false,has_sources,physics::MurphyKoop,"p3::prevent_liq_supersaturation"); //"false" means NOT sat w/ respect to ice @@ -46,18 +48,18 @@ void Functions::prevent_liq_supersaturation(const Spack& pres, const Spack& //The balance we seek is: // qv-qv_sinks*dt+qv_sources*frac*dt=qsl+dqsl_dT*(T correction due to conservation) // where the T correction for conservation is: - // dt*[latent_heat_sublim/cp*(qi2qv_sublim_tend-frac*qi2qv_sublim_tend) - // +latent_heat_vapor/cp*(qr2qv_evap_tend -frac*qr2qv_evap_tend)] - // =(1-frac)*dt/cp*(latent_heat_sublim*qi2qv_sublim_tend + latent_heat_vap*qr2qv_evap_tend). + // dt*[(latvap+latice)/cp*(qi2qv_sublim_tend-frac*qi2qv_sublim_tend) + // +latvap/cp*(qr2qv_evap_tend -frac*qr2qv_evap_tend)] + // =(1-frac)*dt/cp*((latvap+latice)*qi2qv_sublim_tend + latvap*qr2qv_evap_tend). // Note T correction is positive because frac *reduces* evaporative cooling. Note as well that // dqsl_dt comes from linearization of qsl around the end-of-step T computed before temperature // correction. dqsl_dt should be computed with respect to *liquid* even though frac also adjusts // sublimation because we want to be saturated with respect to liquid at the end of the step. - // dqsl_dt=Latent_heat_vapor*qsl/rv*T^2 following Clausius Clapeyron. Combining and solving for + // dqsl_dt=latvap*qsl/rv*T^2 following Clausius Clapeyron. Combining and solving for // frac yields: - A.set(has_sources,latent_heat_vapor*qsl*dt*inv_cp/(rv*T_endstep*T_endstep) - * (latent_heat_sublim*qi2qv_sublim_tend + latent_heat_vapor*qr2qv_evap_tend) ); + A.set(has_sources,latvap*qsl*dt*inv_cp/(rv*T_endstep*T_endstep) + * ((latvap+latice)*qi2qv_sublim_tend + latvap*qr2qv_evap_tend) ); frac.set(has_sources, (qsl-qv+qv_sinks*dt + A)/(qv_sources*dt + A) ); diff --git a/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp index 3f8bcab2cd51..86feaa04f499 100644 --- a/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_update_prognostics_impl.hpp @@ -15,7 +15,7 @@ ::update_prognostic_ice( const Spack& qr2qi_immers_freeze_tend, const Spack& nr2ni_immers_freeze_tend, const Spack& nr_ice_shed_tend, const Spack& qi2qr_melt_tend, const Spack& ni2nr_melt_tend, const Spack& qi2qv_sublim_tend, const Spack& qv2qi_vapdep_tend, const Spack& qv2qi_nucleat_tend, const Spack& ni_nucleat_tend, const Spack& ni_selfcollect_tend, const Spack& ni_sublim_tend, const Spack& qc2qi_berg_tend, - const Spack& inv_exner, const Spack& latent_heat_sublim, const Spack& latent_heat_fusion, const bool do_predict_nc, + const Spack& inv_exner, const bool do_predict_nc, const Smask& log_wetgrowth, const Scalar dt, const Scalar& nmltratio, const Spack& rho_qm_cloud, Spack& th_atm, Spack& qv, Spack& qi, Spack& ni, Spack& qm, Spack& bm, Spack& qc, Spack& nc, Spack& qr, Spack& nr, @@ -23,6 +23,8 @@ ::update_prognostic_ice( { constexpr Scalar QSMALL = C::QSMALL; constexpr Scalar INV_RHO_RIMEMAX = C::INV_RHO_RIMEMAX; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; qc.set(context, qc + (-qc2qi_hetero_freeze_tend-qc2qi_collect_tend-qc2qr_ice_shed_tend-qc2qi_berg_tend)*dt); if ( do_predict_nc ){ @@ -76,9 +78,9 @@ ::update_prognostic_ice( qv.set(context, qv + (-qv2qi_vapdep_tend+qi2qv_sublim_tend-qv2qi_nucleat_tend)*dt); constexpr Scalar INV_CP = C::INV_CP; - th_atm.set(context, th_atm + inv_exner * ((qv2qi_vapdep_tend - qi2qv_sublim_tend + qv2qi_nucleat_tend) * latent_heat_sublim * INV_CP + - (qr2qi_collect_tend + qc2qi_collect_tend + qc2qi_hetero_freeze_tend + qr2qi_immers_freeze_tend - - qi2qr_melt_tend + qc2qi_berg_tend) * latent_heat_fusion * INV_CP) * dt); + th_atm.set(context, th_atm + inv_exner * ((qv2qi_vapdep_tend - qi2qv_sublim_tend + qv2qi_nucleat_tend) * (latvap+latice) * INV_CP + + (qr2qi_collect_tend + qc2qi_collect_tend + qc2qi_hetero_freeze_tend + qr2qi_immers_freeze_tend - + qi2qr_melt_tend + qc2qi_berg_tend) * latice * INV_CP) * dt); } template @@ -88,13 +90,14 @@ ::update_prognostic_liquid( const Spack& qc2qr_accret_tend, const Spack& nc_accret_tend, const Spack& qc2qr_autoconv_tend,const Spack& nc2nr_autoconv_tend, const Spack& ncautr, const Spack& nc_selfcollect_tend, const Spack& qr2qv_evap_tend, const Spack& nr_evap_tend, const Spack& nr_selfcollect_tend, - const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Spack& latent_heat_vapor, + const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Scalar dt, Spack& th_atm, Spack& qv, Spack& qc, Spack& nc, Spack& qr, Spack& nr, const Smask& context) { constexpr Scalar NCCNST = C::NCCNST; constexpr int IPARAM = C::IPARAM; constexpr Scalar INV_CP = C::INV_CP; + constexpr Scalar latvap = C::LatVap; qc.set(context, qc + (-qc2qr_accret_tend-qc2qr_autoconv_tend)*dt); qr.set(context, qr + (qc2qr_accret_tend+qc2qr_autoconv_tend-qr2qv_evap_tend)*dt); @@ -115,7 +118,7 @@ ::update_prognostic_liquid( qv.set(context, qv + qr2qv_evap_tend *dt); - th_atm.set(context, th_atm + inv_exner*(-qr2qv_evap_tend * latent_heat_vapor * INV_CP) * dt); + th_atm.set(context, th_atm + inv_exner*(-qr2qv_evap_tend * latvap * INV_CP) * dt); } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 1749a4ae7346..81a492688007 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -191,6 +191,10 @@ struct Functions view_2d precip_liq_flux; // Grid-box average ice/snow flux [kg m^-2 s^-1] pverp view_2d precip_ice_flux; + // Total precipitation (rain + snow) [kg/kg/s] + view_2d precip_total_tend; + // Evaporation of total precipitation (rain + snow) [kg/kg/s] + view_2d nevapr; }; // This struct stores time stepping and grid-index-related information. @@ -242,6 +246,45 @@ struct Functions view_dnu_table dnu_table_vals; }; +#ifdef SCREAM_P3_SMALL_KERNELS + struct P3Temporaries { + P3Temporaries() = default; + // shape parameter of rain + view_2d mu_r; + // temperature at the beginning of the microphysics step [K] + view_2d T_atm; + // 2D size distribution and fallspeed parameters + view_2d lamr, logn0r, nu; + view_2d cdist, cdist1, cdistr; + // Variables needed for in-cloud calculations + // Inverse cloud fractions (1/cld) + view_2d inv_cld_frac_i, inv_cld_frac_l, inv_cld_frac_r; + // In cloud mass-mixing ratios + view_2d qc_incld, qr_incld, qi_incld, qm_incld; + // In cloud number concentrations + view_2d nc_incld, nr_incld, ni_incld, bm_incld; + // Other + view_2d inv_dz, inv_rho, ze_ice, ze_rain; + view_2d prec, rho, rhofacr, rhofaci; + view_2d acn, qv_sat_l, qv_sat_i, sup; + view_2d qv_supersat_i, tmparr2, exner; + view_2d diag_equiv_reflectivity, diag_vm_qi, diag_diam_qi; + view_2d pratot, prctot; + // p3_tend_out, may not need these + view_2d qtend_ignore, ntend_ignore; + // Variables still used in F90 but removed from C++ interface + view_2d mu_c, lamc; + view_2d qr_evap_tend; + // cloud sedimentation + view_2d v_qc, v_nc, flux_qx, flux_nx; + // ice sedimentation + view_2d v_qit, v_nit, flux_nit, flux_bir; + view_2d flux_qir, flux_qit; + // rain sedimentation + view_2d v_qr, v_nr; + }; +#endif + // -- Table3 -- struct Table3 { @@ -416,7 +459,7 @@ struct Functions const uview_1d& nc_tend, Scalar& precip_liq_surf); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void cloud_sedimentation_disp( const uview_2d& qc_incld, const uview_2d& rho, @@ -464,7 +507,7 @@ struct Functions Scalar& precip_liq_surf, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void rain_sedimentation_disp( const uview_2d& rho, const uview_2d& inv_rho, @@ -514,7 +557,7 @@ struct Functions Scalar& precip_ice_surf, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void ice_sedimentation_disp( const uview_2d& rho, const uview_2d& inv_rho, @@ -545,7 +588,6 @@ struct Functions static void homogeneous_freezing( const uview_1d& T_atm, const uview_1d& inv_exner, - const uview_1d& latent_heat_fusion, const MemberType& team, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_1d& qc, @@ -558,11 +600,10 @@ struct Functions const uview_1d& bm, const uview_1d& th_atm); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void homogeneous_freezing_disp( const uview_2d& T_atm, const uview_2d& inv_exner, - const uview_2d& latent_heat_fusion, const Int& nj, const Int& nk, const Int& ktop, const Int& kbot, const Int& kdir, const uview_2d& qc, const uview_2d& nc, @@ -724,8 +765,8 @@ struct Functions const Spack& qr2qi_collect_tend, const Spack& nr_collect_tend, const Spack& qr2qi_immers_freeze_tend, const Spack& nr2ni_immers_freeze_tend, const Spack& nr_ice_shed_tend, const Spack& qi2qr_melt_tend, const Spack& ni2nr_melt_tend, const Spack& qi2qv_sublim_tend, const Spack& qv2qi_vapdep_tend, const Spack& qv2qi_nucleat_tend, const Spack& ni_nucleat_tend, const Spack& ni_selfcollect_tend, - const Spack& ni_sublim_tend, const Spack& qc2qi_berg_tend, const Spack& inv_exner, const Spack& latent_heat_sublim, - const Spack& latent_heat_fusion, const bool do_predict_nc, const Smask& log_wetgrowth, const Scalar dt, + const Spack& ni_sublim_tend, const Spack& qc2qi_berg_tend, const Spack& inv_exner, + const bool do_predict_nc, const Smask& log_wetgrowth, const Scalar dt, const Scalar& nmltratio, const Spack& rho_qm_cloud, Spack& th_atm, Spack& qv, Spack& qi, Spack& ni, Spack& qm, Spack& bm, Spack& qc, Spack& nc, Spack& qr, Spack& nr, const Smask& context = Smask(true)); @@ -733,7 +774,6 @@ struct Functions // TODO (comments) KOKKOS_FUNCTION static void get_time_space_phys_variables(const Spack& T_atm, const Spack& pres, const Spack& rho, - const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, const Spack& qv_sat_l, const Spack& qv_sat_i, Spack& mu, Spack& dv, Spack& sc, Spack& dqsdt, Spack& dqsidt, Spack& ab, Spack& abi, Spack& kap, Spack& eii, @@ -792,14 +832,14 @@ struct Functions const Spack& cld_frac_l, const Spack& cld_frac_r, const Spack& qv, const Spack& qv_prev, const Spack& qv_sat_l, const Spack& qv_sat_i, const Spack& ab, const Spack& abi, const Spack& epsr, const Spack & epsi_tot, const Spack& t, const Spack& t_prev, - const Spack& latent_heat_sublim, const Spack& dqsdt, const Scalar& dt, + const Spack& dqsdt, const Scalar& dt, Spack& qr2qv_evap_tend, Spack& nr_evap_tend, const Smask& context = Smask(true)); //get number and mass tendencies due to melting ice KOKKOS_FUNCTION static void ice_melting(const Spack& rho, const Spack& T_atm, const Spack& pres, const Spack& rhofaci, - const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, + const Spack& table_val_qi2qr_melting, const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& sc, const Spack& mu, const Spack& kap, const Spack& qv, const Spack& qi_incld, const Spack& ni_incld, Spack& qi2qr_melt_tend, Spack& ni2nr_melt_tend, const Smask& context = Smask(true)); @@ -809,7 +849,7 @@ struct Functions static void update_prognostic_liquid(const Spack& qc2qr_accret_tend, const Spack& nc_accret_tend, const Spack& qc2qr_autoconv_tend,const Spack& nc2nr_autoconv_tend, const Spack& ncautr, const Spack& nc_selfcollect_tend, const Spack& qr2qv_evap_tend, const Spack& nr_evap_tend, const Spack& nr_selfcollect_tend, - const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Spack& latent_heat_vapor, + const bool do_predict_nc, const bool do_prescribed_CCN, const Spack& inv_rho, const Spack& inv_exner, const Scalar dt, Spack& th_atm, Spack& qv, Spack& qc, Spack& nc, Spack& qr, Spack& nr, const Smask& context = Smask(true)); @@ -850,21 +890,18 @@ struct Functions KOKKOS_FUNCTION static void ice_cldliq_wet_growth(const Spack& rho, const Spack& temp, const Spack& pres, const Spack& rhofaci, const Spack& table_val_qi2qr_melting, - const Spack& table_val_qi2qr_vent_melt, const Spack& latent_heat_vapor, const Spack& latent_heat_fusion, const Spack& dv, + const Spack& table_val_qi2qr_vent_melt, const Spack& dv, const Spack& kap, const Spack& mu, const Spack& sc, const Spack& qv, const Spack& qc_incld, const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Smask& log_wetgrowth, Spack& qr2qi_collect_tend, Spack& qc2qi_collect_tend, Spack& qc_growth_rate, Spack& nr_ice_shed_tend, Spack& qc2qr_ice_shed_tend, const Smask& context = Smask(true)); - // Note: not a kernel function - static void get_latent_heat(const Int& nj, const Int& nk, view_2d& v, view_2d& s, view_2d& f); - KOKKOS_FUNCTION static void check_values(const uview_1d& qv, const uview_1d& temp, const Int& ktop, const Int& kbot, const Int& timestepcount, const bool& force_abort, const Int& source_ind, const MemberType& team, const uview_1d& col_loc); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void check_values_disp(const uview_2d& qv, const uview_2d& temp, const Int& ktop, const Int& kbot, const Int& timestepcount, const bool& force_abort, const Int& source_ind, const uview_2d& col_loc, const Int& nj, const Int& nk); @@ -910,7 +947,7 @@ struct Functions Scalar& precip_ice_surf, view_1d_ptr_array& zero_init); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_init_disp( const Int& nj,const Int& nk_pack, const uview_2d& cld_frac_i, const uview_2d& cld_frac_l, @@ -950,9 +987,6 @@ struct Functions const uview_1d& inv_cld_frac_l, const uview_1d& inv_cld_frac_i, const uview_1d& inv_cld_frac_r, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& T_atm, const uview_1d& rho, const uview_1d& inv_rho, @@ -984,7 +1018,7 @@ struct Functions bool& is_hydromet_present, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part1_disp( const Int& nj, const Int& nk, @@ -1001,9 +1035,6 @@ struct Functions const uview_2d& inv_cld_frac_l, const uview_2d& inv_cld_frac_i, const uview_2d& inv_cld_frac_r, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& T_atm, const uview_2d& rho, const uview_2d& inv_rho, @@ -1084,9 +1115,6 @@ struct Functions const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, - const uview_1d& latent_heat_fusion, const uview_1d& qc_incld, const uview_1d& qr_incld, const uview_1d& qi_incld, @@ -1117,7 +1145,7 @@ struct Functions const Int& nk, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part2_disp( const Int& nj, const Int& nk, @@ -1165,9 +1193,6 @@ struct Functions const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, - const uview_2d& latent_heat_fusion, const uview_2d& qc_incld, const uview_2d& qr_incld, const uview_2d& qi_incld, @@ -1223,8 +1248,6 @@ struct Functions const uview_1d& ni, const uview_1d& qm, const uview_1d& bm, - const uview_1d& latent_heat_vapor, - const uview_1d& latent_heat_sublim, const uview_1d& mu_c, const uview_1d& nu, const uview_1d& lamc, @@ -1242,7 +1265,7 @@ struct Functions const uview_1d& diag_eff_radius_qr, const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static void p3_main_part3_disp( const Int& nj, const Int& nk_pack, @@ -1266,8 +1289,6 @@ struct Functions const uview_2d& ni, const uview_2d& qm, const uview_2d& bm, - const uview_2d& latent_heat_vapor, - const uview_2d& latent_heat_sublim, const uview_2d& mu_c, const uview_2d& nu, const uview_2d& lamc, @@ -1297,6 +1318,9 @@ struct Functions const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + const P3Temporaries& temporaries, +#endif const WorkspaceManager& workspace_mgr, Int nj, // number of columns Int nk, // number of vertical cells per column @@ -1315,7 +1339,7 @@ struct Functions Int nk, // number of vertical cells per column const physics::P3_Constants & p3constants); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_P3_SMALL_KERNELS static Int p3_main_internal_disp( const P3Runtime& runtime_options, const P3PrognosticState& prognostic_state, @@ -1324,6 +1348,7 @@ struct Functions const P3Infrastructure& infrastructure, const P3HistoryOnly& history_only, const P3LookupTables& lookup_tables, + const P3Temporaries& temporaries, const WorkspaceManager& workspace_mgr, Int nj, // number of columns Int nk, // number of vertical cells per column @@ -1331,7 +1356,7 @@ struct Functions #endif KOKKOS_FUNCTION - static void ice_supersat_conservation(Spack& qidep, Spack& qinuc, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& latent_heat_sublim, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context = Smask(true)); + static void ice_supersat_conservation(Spack& qidep, Spack& qinuc, const Spack& cld_frac_i, const Spack& qv, const Spack& qv_sat_i, const Spack& t_atm, const Real& dt, const Spack& qi2qv_sublim_tend, const Spack& qr2qv_evap_tend, const Smask& context = Smask(true)); KOKKOS_FUNCTION static void nc_conservation(const Spack& nc, const Spack& nc_selfcollect_tend, const Real& dt, Spack& nc_collect_tend, Spack& nc2ni_immers_freeze_tend, Spack& nc_accret_tend, Spack& nc2nr_autoconv_tend, const Smask& context = Smask(true)); @@ -1343,7 +1368,7 @@ struct Functions static void ni_conservation(const Spack& ni, const Spack& ni_nucleat_tend, const Spack& nr2ni_immers_freeze_tend, const Spack& nc2ni_immers_freeze_tend, const Real& dt, Spack& ni2nr_melt_tend, Spack& ni_sublim_tend, Spack& ni_selfcollect_tend, const Smask& context = Smask(true)); KOKKOS_FUNCTION - static void prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Spack& latent_heat_vapor, const Spack& latent_heat_sublim, const Scalar& dt, const Spack& qidep, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context = Smask(true) ); + static void prevent_liq_supersaturation(const Spack& pres, const Spack& t_atm, const Spack& qv, const Scalar& dt, const Spack& qidep, const Spack& qinuc, Spack& qi2qv_sublim_tend, Spack& qr2qv_evap_tend, const Smask& context = Smask(true) ); }; // struct Functions template @@ -1391,7 +1416,6 @@ void init_tables_from_f90_c(Real* vn_table_vals_data, Real* vm_table_vals_data, # include "p3_ice_melting_impl.hpp" # include "p3_calc_liq_relaxation_timescale_impl.hpp" # include "p3_ice_cldliq_wet_growth_impl.hpp" -# include "p3_get_latent_heat_impl.hpp" # include "p3_check_values_impl.hpp" # include "p3_incloud_mixingratios_impl.hpp" # include "p3_subgrid_variance_scaling_impl.hpp" diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index 83aeb5b16d3d..40d82d4f6529 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -1417,7 +1417,7 @@ void rain_sedimentation_f( void homogeneous_freezing_f( Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* T_atm, Real* inv_exner, Real* latent_heat_fusion, + Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm) { using P3F = Functions; @@ -1440,31 +1440,31 @@ void homogeneous_freezing_f( const Int nk_pack = ekat::npack(nk); // Set up views - std::vector temp_d(HomogeneousFreezingData::NUM_ARRAYS); + std::vector temp_d(HomogeneousFreezingData::NUM_ARRAYS-1); - ekat::host_to_device({T_atm, inv_exner, latent_heat_fusion, qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, + ekat::host_to_device({T_atm, inv_exner, qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, nk, temp_d); + int current_index = 0; view_1d - t_d (temp_d[0]), - inv_exner_d (temp_d[1]), - latent_heat_fusion_d (temp_d[2]), - qc_d (temp_d[3]), - nc_d (temp_d[4]), - qr_d (temp_d[5]), - nr_d (temp_d[6]), - qi_d (temp_d[7]), - ni_d (temp_d[8]), - qm_d (temp_d[9]), - bm_d (temp_d[10]), - th_atm_d (temp_d[11]); + t_d (temp_d[current_index++]), + inv_exner_d (temp_d[current_index++]), + qc_d (temp_d[current_index++]), + nc_d (temp_d[current_index++]), + qr_d (temp_d[current_index++]), + nr_d (temp_d[current_index++]), + qi_d (temp_d[current_index++]), + ni_d (temp_d[current_index++]), + qm_d (temp_d[current_index++]), + bm_d (temp_d[current_index++]), + th_atm_d (temp_d[current_index++]); // Call core function from kernel auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, nk_pack); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { P3F::homogeneous_freezing( - t_d, inv_exner_d, latent_heat_fusion_d, + t_d, inv_exner_d, team, nk, ktop, kbot, kdir, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, th_atm_d); @@ -1476,37 +1476,6 @@ void homogeneous_freezing_f( ekat::device_to_host({qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, nk, inout_views); } -void get_latent_heat_f(Int its, Int ite, Int kts, Int kte, Real* v, Real* s, Real* f) -{ - using P3F = Functions; - using Spack = typename P3F::Spack; - using view_2d = typename P3F::view_2d; - - EKAT_REQUIRE_MSG(kte >= kts, - "kte must be >= kts, kts=" << kts << " kte=" << kte); - - EKAT_REQUIRE_MSG(ite >= its, - "ite must be >= its, its=" << its << " ite=" << ite); - - kts -= 1; - kte -= 1; - its -= 1; - ite -= 1; - - Int nk = (kte - kts) + 1; - Int nj = (ite - its) + 1; - - // Set up views - view_2d v_d("v_d", nj, nk), - s_d("s_d", nj, nk), - f_d("f_d", nj, nk); - - P3F::get_latent_heat(nj, nk, v_d, s_d, f_d); - - std::vector out_views = {v_d, s_d, f_d}; - ekat::device_to_host({v, s, f}, nj, nk, out_views, true); -} - void check_values_f(Real* qv, Real* temp, Int kstart, Int kend, Int timestepcount, bool force_abort, Int source_ind, Real* col_loc) { @@ -1545,7 +1514,7 @@ void p3_main_part1_f( bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, + Real* inv_cld_frac_r, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, @@ -1576,7 +1545,7 @@ void p3_main_part1_f( ekat::host_to_device({pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, - acn, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, + acn, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, nccn_prescribed}, nk, temp_d); @@ -1609,18 +1578,15 @@ void p3_main_part1_f( ni_d (temp_d[25]), qm_d (temp_d[26]), bm_d (temp_d[27]), - latent_heat_vapor_d (temp_d[28]), - latent_heat_sublim_d (temp_d[29]), - latent_heat_fusion_d (temp_d[30]), - qc_incld_d (temp_d[31]), - qr_incld_d (temp_d[32]), - qi_incld_d (temp_d[33]), - qm_incld_d (temp_d[34]), - nc_incld_d (temp_d[35]), - nr_incld_d (temp_d[36]), - ni_incld_d (temp_d[37]), - bm_incld_d (temp_d[38]), - nccn_prescribed_d (temp_d[39]); + qc_incld_d (temp_d[28]), + qr_incld_d (temp_d[29]), + qi_incld_d (temp_d[30]), + qm_incld_d (temp_d[31]), + nc_incld_d (temp_d[32]), + nr_incld_d (temp_d[33]), + ni_incld_d (temp_d[34]), + bm_incld_d (temp_d[35]), + nccn_prescribed_d (temp_d[36]); // Call core function from kernel bview_1d bools_d("bools", 2); @@ -1630,7 +1596,7 @@ void p3_main_part1_f( P3F::p3_main_part1( team, nk, do_predict_nc, do_prescribed_CCN, dt, pres_d, dpres_d, dz_d, nc_nuceat_tend_d, nccn_prescribed_d, inv_exner_d, exner_d, inv_cld_frac_l_d, inv_cld_frac_i_d, - inv_cld_frac_r_d, latent_heat_vapor_d, latent_heat_sublim_d, latent_heat_fusion_d, + inv_cld_frac_r_d, t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, @@ -1661,7 +1627,7 @@ void p3_main_part2_f( Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, - Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, + Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, Real* mu_c, Real* nu, Real* lamc, Real* cdist, Real* cdist1, Real* cdistr, Real* mu_r, Real* lamr, Real* logn0r, Real* qv2qi_depos_tend, Real* precip_total_tend, Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, Real* prctot, bool* is_hydromet_present) @@ -1688,82 +1654,80 @@ void p3_main_part2_f( const Real max_total_ni = 740.0e3; // Hard-code this value for F90 comparison // Set up views - std::vector temp_d(P3MainPart2Data::NUM_ARRAYS); + std::vector temp_d(P3MainPart2Data::NUM_ARRAYS-3); ekat::host_to_device({pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, - qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, + qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, pratot, prctot, qv_prev, t_prev }, nk, temp_d); + int current_index = 0; view_1d - pres_d (temp_d[0]), - dpres_d (temp_d[1]), - dz_d (temp_d[2]), - nc_nuceat_tend_d (temp_d[3]), - inv_exner_d (temp_d[4]), - exner_d (temp_d[5]), - inv_cld_frac_l_d (temp_d[6]), - inv_cld_frac_i_d (temp_d[7]), - inv_cld_frac_r_d (temp_d[8]), - ni_activated_d (temp_d[9]), - inv_qc_relvar_d (temp_d[10]), - cld_frac_i_d (temp_d[11]), - cld_frac_l_d (temp_d[12]), - cld_frac_r_d (temp_d[13]), - t_d (temp_d[14]), - rho_d (temp_d[15]), - inv_rho_d (temp_d[16]), - qv_sat_l_d (temp_d[17]), - qv_sat_i_d (temp_d[18]), - qv_supersat_i_d (temp_d[19]), - rhofacr_d (temp_d[20]), - rhofaci_d (temp_d[21]), - acn_d (temp_d[22]), - qv_d (temp_d[23]), - th_atm_d (temp_d[24]), - qc_d (temp_d[25]), - nc_d (temp_d[26]), - qr_d (temp_d[27]), - nr_d (temp_d[28]), - qi_d (temp_d[29]), - ni_d (temp_d[30]), - qm_d (temp_d[31]), - bm_d (temp_d[32]), - latent_heat_vapor_d (temp_d[33]), - latent_heat_sublim_d(temp_d[34]), - latent_heat_fusion_d(temp_d[35]), - qc_incld_d (temp_d[36]), - qr_incld_d (temp_d[37]), - qi_incld_d (temp_d[38]), - qm_incld_d (temp_d[39]), - nc_incld_d (temp_d[40]), - nr_incld_d (temp_d[41]), - ni_incld_d (temp_d[42]), - bm_incld_d (temp_d[43]), - mu_c_d (temp_d[44]), - nu_d (temp_d[45]), - lamc_d (temp_d[46]), - cdist_d (temp_d[47]), - cdist1_d (temp_d[48]), - cdistr_d (temp_d[49]), - mu_r_d (temp_d[50]), - lamr_d (temp_d[51]), - logn0r_d (temp_d[52]), - qv2qi_depos_tend_d (temp_d[53]), - precip_total_tend_d (temp_d[54]), - nevapr_d (temp_d[55]), - qr_evap_tend_d (temp_d[56]), - vap_liq_exchange_d (temp_d[57]), - vap_ice_exchange_d (temp_d[58]), - liq_ice_exchange_d (temp_d[59]), - pratot_d (temp_d[60]), - prctot_d (temp_d[61]), - qv_prev_d (temp_d[62]), - t_prev_d (temp_d[63]); + pres_d (temp_d[current_index++]), + dpres_d (temp_d[current_index++]), + dz_d (temp_d[current_index++]), + nc_nuceat_tend_d (temp_d[current_index++]), + inv_exner_d (temp_d[current_index++]), + exner_d (temp_d[current_index++]), + inv_cld_frac_l_d (temp_d[current_index++]), + inv_cld_frac_i_d (temp_d[current_index++]), + inv_cld_frac_r_d (temp_d[current_index++]), + ni_activated_d (temp_d[current_index++]), + inv_qc_relvar_d (temp_d[current_index++]), + cld_frac_i_d (temp_d[current_index++]), + cld_frac_l_d (temp_d[current_index++]), + cld_frac_r_d (temp_d[current_index++]), + t_d (temp_d[current_index++]), + rho_d (temp_d[current_index++]), + inv_rho_d (temp_d[current_index++]), + qv_sat_l_d (temp_d[current_index++]), + qv_sat_i_d (temp_d[current_index++]), + qv_supersat_i_d (temp_d[current_index++]), + rhofacr_d (temp_d[current_index++]), + rhofaci_d (temp_d[current_index++]), + acn_d (temp_d[current_index++]), + qv_d (temp_d[current_index++]), + th_atm_d (temp_d[current_index++]), + qc_d (temp_d[current_index++]), + nc_d (temp_d[current_index++]), + qr_d (temp_d[current_index++]), + nr_d (temp_d[current_index++]), + qi_d (temp_d[current_index++]), + ni_d (temp_d[current_index++]), + qm_d (temp_d[current_index++]), + bm_d (temp_d[current_index++]), + qc_incld_d (temp_d[current_index++]), + qr_incld_d (temp_d[current_index++]), + qi_incld_d (temp_d[current_index++]), + qm_incld_d (temp_d[current_index++]), + nc_incld_d (temp_d[current_index++]), + nr_incld_d (temp_d[current_index++]), + ni_incld_d (temp_d[current_index++]), + bm_incld_d (temp_d[current_index++]), + mu_c_d (temp_d[current_index++]), + nu_d (temp_d[current_index++]), + lamc_d (temp_d[current_index++]), + cdist_d (temp_d[current_index++]), + cdist1_d (temp_d[current_index++]), + cdistr_d (temp_d[current_index++]), + mu_r_d (temp_d[current_index++]), + lamr_d (temp_d[current_index++]), + logn0r_d (temp_d[current_index++]), + qv2qi_depos_tend_d (temp_d[current_index++]), + precip_total_tend_d (temp_d[current_index++]), + nevapr_d (temp_d[current_index++]), + qr_evap_tend_d (temp_d[current_index++]), + vap_liq_exchange_d (temp_d[current_index++]), + vap_ice_exchange_d (temp_d[current_index++]), + liq_ice_exchange_d (temp_d[current_index++]), + pratot_d (temp_d[current_index++]), + prctot_d (temp_d[current_index++]), + qv_prev_d (temp_d[current_index++]), + t_prev_d (temp_d[current_index++]); // Call core function from kernel const auto dnu = P3GlobalForFortran::dnu(); @@ -1780,7 +1744,7 @@ void p3_main_part2_f( inv_cld_frac_i_d, inv_cld_frac_r_d, ni_activated_d, inv_qc_relvar_d, cld_frac_i_d, cld_frac_l_d, cld_frac_r_d, qv_prev_d, t_prev_d, t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, - latent_heat_vapor_d, latent_heat_sublim_d, latent_heat_fusion_d, qc_incld_d, qr_incld_d, qi_incld_d, + qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, mu_c_d, nu_d, lamc_d, cdist_d, cdist1_d, cdistr_d, mu_r_d, lamr_d, logn0r_d, qv2qi_depos_tend_d, precip_total_tend_d, nevapr_d, qr_evap_tend_d, vap_liq_exchange_d, @@ -1791,7 +1755,7 @@ void p3_main_part2_f( std::vector inout_views = { t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, - latent_heat_vapor_d, latent_heat_sublim_d, latent_heat_fusion_d, qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, + qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, mu_c_d, nu_d, lamc_d, cdist_d, cdist1_d, cdistr_d, mu_r_d, lamr_d, logn0r_d, qv2qi_depos_tend_d, precip_total_tend_d, nevapr_d, qr_evap_tend_d, vap_liq_exchange_d, vap_ice_exchange_d, @@ -1800,7 +1764,7 @@ void p3_main_part2_f( ekat::device_to_host({ T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th_atm, qc, nc, - qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, + qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, @@ -1818,7 +1782,7 @@ void p3_main_part3_f( Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, - Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* mu_c, Real* nu, Real* lamc, + Real* bm, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr) @@ -1844,50 +1808,49 @@ void p3_main_part3_f( const Real max_total_ni = 740.0e3; // Hard-code this value for F90 comparison // Set up views - std::vector temp_d(P3MainPart3Data::NUM_ARRAYS); + std::vector temp_d(P3MainPart3Data::NUM_ARRAYS-2); ekat::host_to_device({ inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, rho, inv_rho, rhofaci, qv, th_atm, qc, - nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, + nc, qr, nr, qi, ni, qm, bm, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr}, nk, temp_d); + int current_index = 0; view_1d - inv_exner_d (temp_d[0]), - cld_frac_l_d (temp_d[1]), - cld_frac_r_d (temp_d[2]), - cld_frac_i_d (temp_d[3]), - rho_d (temp_d[4]), - inv_rho_d (temp_d[5]), - rhofaci_d (temp_d[6]), - qv_d (temp_d[7]), - th_atm_d (temp_d[8]), - qc_d (temp_d[9]), - nc_d (temp_d[10]), - qr_d (temp_d[11]), - nr_d (temp_d[12]), - qi_d (temp_d[13]), - ni_d (temp_d[14]), - qm_d (temp_d[15]), - bm_d (temp_d[16]), - latent_heat_vapor_d (temp_d[17]), - latent_heat_sublim_d (temp_d[18]), - mu_c_d (temp_d[19]), - nu_d (temp_d[20]), - lamc_d (temp_d[21]), - mu_r_d (temp_d[22]), - lamr_d (temp_d[23]), - vap_liq_exchange_d (temp_d[24]), - ze_rain_d (temp_d[25]), - ze_ice_d (temp_d[26]), - diag_vm_qi_d (temp_d[27]), - diag_eff_radius_qi_d (temp_d[28]), - diag_diam_qi_d (temp_d[29]), - rho_qi_d (temp_d[30]), - diag_equiv_reflectivity_d (temp_d[31]), - diag_eff_radius_qc_d (temp_d[32]), - diag_eff_radius_qr_d (temp_d[33]); + inv_exner_d (temp_d[current_index++]), + cld_frac_l_d (temp_d[current_index++]), + cld_frac_r_d (temp_d[current_index++]), + cld_frac_i_d (temp_d[current_index++]), + rho_d (temp_d[current_index++]), + inv_rho_d (temp_d[current_index++]), + rhofaci_d (temp_d[current_index++]), + qv_d (temp_d[current_index++]), + th_atm_d (temp_d[current_index++]), + qc_d (temp_d[current_index++]), + nc_d (temp_d[current_index++]), + qr_d (temp_d[current_index++]), + nr_d (temp_d[current_index++]), + qi_d (temp_d[current_index++]), + ni_d (temp_d[current_index++]), + qm_d (temp_d[current_index++]), + bm_d (temp_d[current_index++]), + mu_c_d (temp_d[current_index++]), + nu_d (temp_d[current_index++]), + lamc_d (temp_d[current_index++]), + mu_r_d (temp_d[current_index++]), + lamr_d (temp_d[current_index++]), + vap_liq_exchange_d (temp_d[current_index++]), + ze_rain_d (temp_d[current_index++]), + ze_ice_d (temp_d[current_index++]), + diag_vm_qi_d (temp_d[current_index++]), + diag_eff_radius_qi_d (temp_d[current_index++]), + diag_diam_qi_d (temp_d[current_index++]), + rho_qi_d (temp_d[current_index++]), + diag_equiv_reflectivity_d (temp_d[current_index++]), + diag_eff_radius_qc_d (temp_d[current_index++]), + diag_eff_radius_qr_d (temp_d[current_index++]); // Call core function from kernel const auto dnu = P3GlobalForFortran::dnu(); @@ -1898,8 +1861,8 @@ void p3_main_part3_f( P3F::p3_main_part3(team, nk_pack, max_total_ni, dnu, ice_table_vals, inv_exner_d, cld_frac_l_d, cld_frac_r_d, cld_frac_i_d, rho_d, inv_rho_d, rhofaci_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, - qi_d, ni_d, qm_d, bm_d, latent_heat_vapor_d, - latent_heat_sublim_d, mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, + qi_d, ni_d, qm_d, bm_d, + mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, vap_liq_exchange_d, ze_rain_d, ze_ice_d, diag_vm_qi_d, diag_eff_radius_qi_d, diag_diam_qi_d, rho_qi_d, diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d, physics::P3_Constants()); @@ -1908,14 +1871,14 @@ void p3_main_part3_f( // Sync back to host std::vector inout_views = { rho_d, inv_rho_d, rhofaci_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, - ni_d, qm_d, bm_d, latent_heat_vapor_d, latent_heat_sublim_d, mu_c_d, nu_d, lamc_d, mu_r_d, + ni_d, qm_d, bm_d, mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, vap_liq_exchange_d, ze_rain_d, ze_ice_d, diag_vm_qi_d, diag_eff_radius_qi_d, diag_diam_qi_d, rho_qi_d, diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d }; ekat::device_to_host({ rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, - latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, + mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr }, @@ -2048,12 +2011,42 @@ Int p3_main_f( inv_exner_d, qv_prev_d, t_prev_d}; P3F::P3DiagnosticOutputs diag_outputs{qv2qi_depos_tend_d, precip_liq_surf_d, precip_ice_surf_d, diag_eff_radius_qc_d, diag_eff_radius_qi_d, diag_eff_radius_qr_d, - rho_qi_d,precip_liq_flux_d, precip_ice_flux_d}; + rho_qi_d,precip_liq_flux_d, precip_ice_flux_d, precip_total_tend_d, nevapr_d}; P3F::P3Infrastructure infrastructure{dt, it, its, ite, kts, kte, do_predict_nc, do_prescribed_CCN, col_location_d}; P3F::P3HistoryOnly history_only{liq_ice_exchange_d, vap_liq_exchange_d, vap_ice_exchange_d}; + const Int nk_pack = ekat::npack(nk); +#ifdef SCREAM_P3_SMALL_KERNELS + view_2d + mu_r("mu_r", nj, nk_pack), T_atm("T_atm", nj, nk_pack), lamr("lamr", nj, nk_pack), logn0r("logn0r", nj, nk_pack), nu("nu", nj, nk_pack), + cdist("cdist", nj, nk_pack), cdist1("cdist1", nj, nk_pack), cdistr("cdistr", nj, nk_pack), inv_cld_frac_i("inv_cld_frac_i", nj, nk_pack), + inv_cld_frac_l("inv_cld_frac_l", nj, nk_pack), inv_cld_frac_r("inv_cld_frac_r", nj, nk_pack), qc_incld("qc_incld", nj, nk_pack), + qr_incld("qr_incld", nj, nk_pack), qi_incld("qi_incld", nj, nk_pack), qm_incld("qm_incld", nj, nk_pack), nc_incld("nc_incld", nj, nk_pack), + nr_incld("nr_incld", nj, nk_pack), ni_incld("ni_incld", nj, nk_pack), bm_incld("bm_incld", nj, nk_pack), inv_dz("inv_dz", nj, nk_pack), + inv_rho("inv_rho", nj, nk_pack), ze_ice("ze_ice", nj, nk_pack), ze_rain("ze_rain", nj, nk_pack), prec("prec", nj, nk_pack), + rho("rho", nj, nk_pack), rhofacr("rhofacr", nj, nk_pack), rhofaci("rhofaci", nj, nk_pack), acn("acn", nj, nk_pack), qv_sat_l("qv_sat", nj, nk_pack), + qv_sat_i("qv_sat_i", nj, nk_pack), sup("sup", nj, nk_pack), qv_supersat_i("qv_supersat", nj, nk_pack), tmparr2("tmparr2", nj, nk_pack), + exner("exner", nj, nk_pack), diag_equiv_reflectivity("diag_equiv_ref", nj, nk_pack), diag_vm_qi("diag_vm_qi", nj, nk_pack), + diag_diam_qi("diag_diam_qi", nj, nk_pack), pratot("pratot", nj, nk_pack), prctot("prctot", nj, nk_pack), qtend_ignore("qtend_ignore", nj, nk_pack), + ntend_ignore("ntend_ignore", nj, nk_pack), mu_c("mu_c", nj, nk_pack), lamc("lamc", nj, nk_pack), qr_evap_tend("qr_evap_tend", nj, nk_pack), + v_qc("v_qc", nj, nk_pack), v_nc("v_nc", nj, nk_pack), flux_qx("flux_qx", nj, nk_pack), flux_nx("flux_nx", nj, nk_pack), v_qit("v_qit", nj, nk_pack), + v_nit("v_nit", nj, nk_pack), flux_nit("flux_nit", nj, nk_pack), flux_bir("flux_bir", nj, nk_pack), flux_qir("flux_qir", nj, nk_pack), + flux_qit("flux_qit", nj, nk_pack), v_qr("v_qr", nj, nk_pack), v_nr("v_nr", nj, nk_pack); + + P3F::P3Temporaries temporaries{ + mu_r, T_atm, lamr, logn0r, nu, cdist, cdist1, cdistr, inv_cld_frac_i, + inv_cld_frac_l, inv_cld_frac_r, qc_incld, qr_incld, qi_incld, qm_incld, + nc_incld, nr_incld, ni_incld, bm_incld, inv_dz, inv_rho, ze_ice, ze_rain, + prec, rho, rhofacr, rhofaci, acn, qv_sat_l, qv_sat_i, sup, qv_supersat_i, + tmparr2, exner, diag_equiv_reflectivity, diag_vm_qi, diag_diam_qi, + pratot, prctot, qtend_ignore, ntend_ignore, mu_c, lamc, qr_evap_tend, + v_qc, v_nc, flux_qx, flux_nx, v_qit, v_nit, flux_nit, flux_bir, flux_qir, + flux_qit, v_qr, v_nr + }; +#endif + // load tables view_1d_table mu_r_table_vals; view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; @@ -2068,12 +2061,15 @@ Int p3_main_f( P3F::P3Runtime runtime_options{740.0e3}; // Create local workspace - const Int nk_pack = ekat::npack(nk); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); ekat::WorkspaceManager workspace_mgr(nk_pack, 52, policy); auto elapsed_microsec = P3F::p3_main(runtime_options, prog_state, diag_inputs, diag_outputs, infrastructure, - history_only, lookup_tables, workspace_mgr, nj, nk, physics::P3_Constants()); + history_only, lookup_tables, +#ifdef SCREAM_P3_SMALL_KERNELS + temporaries, +#endif + workspace_mgr, nj, nk, physics::P3_Constants()); Kokkos::parallel_for(nj, KOKKOS_LAMBDA(const Int& i) { precip_liq_surf_temp_d(0, i / Spack::n)[i % Spack::n] = precip_liq_surf_d(i); diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/p3_functions_f90.hpp index 926c9ab47017..d1c29d3de0de 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.hpp @@ -908,11 +908,9 @@ void rain_sedimentation_f( void homogeneous_freezing_f( Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* T_atm, Real* inv_exner, Real* latent_heat_fusion, + Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm); -void get_latent_heat_f(Int its, Int ite, Int kts, Int kte, Real* v, Real* s, Real* f); - void check_values_f(Real* Qv, Real* temp, Int kstart, Int kend, Int timestepcount, bool force_abort, Int source_ind, Real* col_loc); @@ -921,7 +919,7 @@ void p3_main_part1_f( bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, + Real* inv_cld_frac_r, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, @@ -931,7 +929,7 @@ void p3_main_part2_f( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, - Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, + Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, Real* mu_c, Real* nu, Real* lamc, Real* cdist, Real* cdist1, Real* cdistr, Real* mu_r, Real* lamr, Real* logn0r, Real* qv2qi_depos_tend, Real* precip_total_tend, Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, Real* prctot, bool* is_hydromet_present); @@ -939,7 +937,7 @@ void p3_main_part2_f( void p3_main_part3_f( Int kts, Int kte, Int kbot, Int ktop, Int kdir, Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, - Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, + Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr); diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index d75af1cb2edb..ea0a18411c10 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -16,7 +16,7 @@ module p3_iso_c contains subroutine append_precision(string, prefix) - character(kind=c_char, len=256), intent(inout) :: string + character(kind=c_char, len=512), intent(out) :: string character(*), intent(in) :: prefix real(kind=c_real) :: s @@ -57,7 +57,7 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) real(kind=c_real), dimension(300,10), target :: vn_table_vals, vm_table_vals, revap_table_vals character(len=256), pointer :: lookup_file_dir - character(kind=c_char, len=256) :: mu_r_filename, revap_filename, vn_filename, vm_filename + character(kind=c_char, len=512) :: mu_r_filename, revap_filename, vn_filename, vm_filename integer :: len logical :: ok character(len=16) :: p3_version="4.1.1" ! TODO: Change to be dependent on table version and path specified in p3_functions.hpp @@ -69,10 +69,17 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) info = 0 ok = .false. - call append_precision(mu_r_filename, SCREAM_DATA_DIR//"/tables/mu_r_table_vals.dat") - call append_precision(revap_filename, SCREAM_DATA_DIR//"/tables/revap_table_vals.dat") - call append_precision(vn_filename, SCREAM_DATA_DIR//"/tables/vn_table_vals.dat") - call append_precision(vm_filename, SCREAM_DATA_DIR//"/tables/vm_table_vals.dat") +#ifdef SCREAM_DOUBLE_PRECISION + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat8'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat8'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat8'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat8'//C_NULL_CHAR +#else + mu_r_filename = lookup_file_dir(1:len)//'/mu_r_table_vals.dat4'//C_NULL_CHAR + revap_filename = lookup_file_dir(1:len)//'/revap_table_vals.dat4'//C_NULL_CHAR + vn_filename = lookup_file_dir(1:len)//'/vn_table_vals.dat4'//C_NULL_CHAR + vm_filename = lookup_file_dir(1:len)//'/vm_table_vals.dat4'//C_NULL_CHAR +#endif if (write_tables) then call p3_init_b() diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 4e8901069e05..217c2945e489 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -26,7 +26,6 @@ set(P3_TESTS_SRCS p3_ice_melting_unit_tests.cpp p3_evaporate_rain_unit_tests.cpp p3_ice_cldliq_wet_growth_unit_tests.cpp - p3_get_latent_heat_unit_tests.cpp p3_subgrid_variance_scaling_unit_tests.cpp p3_check_values_unit_tests.cpp p3_incloud_mixingratios_unit_tests.cpp @@ -60,7 +59,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) LABELS "p3;physics;fail" ${FORCE_RUN_DIFF_FAILS}) - if (NOT SCREAM_SMALL_KERNELS) + if (NOT SCREAM_P3_SMALL_KERNELS) CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" LIBS p3_sk THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} diff --git a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp index 24cc3a848188..f8398135a19e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp @@ -82,7 +82,6 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Spack epsi_tot(1./60.); Spack t(287); Spack t_prev(285); - Spack latent_heat_sublim(3.34e5); Spack dqsdt(1e-3); Scalar dt=60; Spack qrtend; @@ -92,7 +91,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip constexpr Scalar QSMALL = C::QSMALL; Functions::evaporate_rain(Spack(QSMALL/2),qc_incld,nr_incld,qi_incld, //qr_incld->QSMALL/2 cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0])<1e-8 ); REQUIRE( std::abs(nrtend[0])<1e-8 ); @@ -101,7 +100,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Spack qr_tiny=Spack(5e-13); Functions::evaporate_rain(qr_tiny,qc_incld,nr_incld,qi_incld, //qr_incld->_tiny cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0] - qr_tiny[0]/dt *(cld_frac_r[0]-cld_frac_l[0])/cld_frac_r[0])<1e-8 ); @@ -111,7 +110,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip //if no rainy areas outside cloud, don't evap Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, cld_frac_r,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, //cld_frac_l->_r - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0])<1e-8 ); REQUIRE( std::abs(nrtend[0])<1e-8 ); @@ -120,7 +119,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, //set qv->qv_sat_l*2 in next line to ensure supersaturated. cld_frac_l,cld_frac_r,qv_sat_l*2,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( std::abs(qrtend[0])<1e-8 ); REQUIRE( std::abs(nrtend[0])<1e-8 ); @@ -129,7 +128,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, //qv -> qv*0.1 to encourage lots of rain evap cld_frac_l,cld_frac_r,qv*0.1,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qrtend,nrtend); REQUIRE( qrtend[0] <= qr_incld[0]/dt); REQUIRE( nrtend[0] <= nr_incld[0]/dt); //keep end-of-step nr positive. Should always be true. @@ -137,6 +136,8 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip }; //end run_property static void run_bfb(){ + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //fortran generated data is input to the following //This subroutine has 20 args, only 18 are supplied here for invoking it as last 2 are intent-outs @@ -152,23 +153,23 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip //rows 9-16: random junk but ensured cld_frac_r>cld_frac_l and subsaturated. EvapRainData espd[max_pack_size] = { //qr_incld, qc_incld, nr_incld, qi_incld, cld_frac_l, cld_frac_r, qv, qv_prev, qv_sat_l, qv_sat_i, ab, abi, epsr, epsi_tot, t, t_prev, lat_ht_sublim, dqsdt, dt - {4.634940e-03,1.215335e-03,6.073270e+07,3.594486e-04,6.134229e-01,9.134229e-01,2.747871e-03,1.911238e-03,5.913313e-03,1.057645e-03,1.782748e+00,1.571392e+00,3.868229e+02,2.248689e+02,3.101180e+02,1.395063e+02,3.335413e+05,5.494606e-03,6.000000e+02}, - {6.175320e-13,4.432407e-03,8.029967e+07,1.905151e-03,2.190099e-01,7.031070e-01,4.172977e-05,7.315360e-03,7.280063e-03,1.378543e-03,1.461443e+00,1.507382e+00,8.452377e+02,1.971876e+02,2.389249e+02,1.497752e+02,3.578466e+05,5.107905e-03,6.000000e+02}, - {4.519798e-03,7.348916e-03,7.420725e+07,2.220971e-03,1.882608e-01,2.934182e-01,4.957590e-03,2.550256e-03,3.136926e-03,4.498115e-03,1.433526e+00,1.207516e+00,9.716844e+02,5.602546e+01,1.389465e+02,1.075863e+02,3.570404e+05,6.771428e-03,6.000000e+02}, - {7.169182e-03,6.657331e-03,9.807967e+07,7.981196e-03,2.914473e-01,6.375719e-01,2.420032e-03,1.223012e-03,7.685516e-03,5.207024e-03,1.644865e+00,1.433872e+00,3.825069e+02,6.550300e+02,1.833466e+02,1.741918e+02,3.295800e+05,3.792982e-03,6.000000e+02}, - {1.103118e-03,9.158125e-03,3.136196e+07,4.286154e-03,2.699078e-01,4.668103e-01,9.645460e-03,6.379119e-03,8.283285e-03,3.342400e-03,1.546698e+00,1.417916e+00,9.289270e+02,9.844129e+02,2.543202e+02,1.932996e+02,3.327786e+05,2.693119e-03,6.000000e+02}, - {4.308000e-03,8.168535e-03,7.439969e+07,5.131497e-03,6.851225e-01,3.298025e-01,4.331812e-03,2.814373e-03,3.592807e-03,1.527499e-03,1.856943e+00,1.003269e+00,9.165690e+02,9.379921e+02,2.163204e+02,3.165814e+02,3.874371e+05,6.801393e-03,6.000000e+02}, - {0.000000e-00,3.318968e-03,4.664041e+07,8.737282e-03,2.585907e-01,6.297295e-02,8.747418e-03,2.710437e-03,2.164895e-03,9.455725e-03,1.241506e+00,1.561393e+00,2.492674e+02,6.546182e+02,2.228772e+02,2.147968e+02,3.590381e+05,5.903261e-03,6.000000e+02}, - {7.677170e-03,6.069057e-05,6.404241e+07,3.094233e-03,3.755403e-01,5.026876e-01,4.723817e-03,1.204228e-03,6.156526e-03,8.194797e-03,1.361509e+00,1.772751e+00,6.420537e+01,4.043364e+02,2.833110e+02,3.314521e+02,3.427004e+05,2.996696e-03,6.000000e+02}, - - {9.999294e-03,3.138400e-03,2.355097e+07,9.897893e-03,7.667177e-01,9.739270e-01,4.221430e-03,3.570130e-03,8.370033e-03,9.527208e-03,1.597218e+00,1.111438e+00,7.832357e+02,8.364566e+02,2.854867e+02,2.340771e+02,3.198709e+05,7.235757e-03,6.000000e+02}, - {8.841793e-03,3.530456e-03,9.618284e+07,9.311658e-03,3.458590e-01,6.978258e-01,1.279864e-03,4.652008e-03,1.869728e-03,8.931663e-03,1.712564e+00,1.223882e+00,9.692403e+02,2.358558e+02,3.204043e+02,1.827677e+02,3.220502e+05,7.646405e-03,6.000000e+02}, - {1.425612e-03,6.653411e-04,2.843806e+07,1.922560e-03,9.100262e-01,0.996264e-01,8.973183e-04,9.857420e-03,6.221419e-03,8.133433e-03,1.815337e+00,1.885506e+00,5.508742e+02,1.612139e+02,2.798523e+02,2.631136e+02,3.045141e+05,4.148666e-03,6.000000e+02}, - {4.125177e-04,4.056163e-03,2.716439e+07,6.484214e-03,1.658752e-01,2.859102e-01,5.724081e-03,6.282997e-03,7.313187e-03,6.049825e-03,1.140910e+00,1.145941e+00,7.490652e+02,5.011633e+02,1.986541e+02,2.745566e+02,3.371001e+05,6.784784e-03,6.000000e+02}, - {5.010628e-03,2.863789e-04,8.953841e+07,3.953058e-03,1.135952e-01,9.718675e-01,1.846157e-03,5.743094e-03,2.842649e-03,8.155366e-03,1.227867e+00,1.894249e+00,1.161776e+02,3.578576e+02,1.240083e+02,1.639791e+02,3.167181e+05,4.497257e-03,6.000000e+02}, - {9.487866e-03,6.584660e-03,6.149682e+06,9.413342e-03,4.757261e-01,6.503885e-01,1.078922e-03,3.489665e-03,3.059596e-03,9.285703e-03,1.192620e+00,1.967205e+00,5.085628e+02,3.741816e+01,1.196252e+02,2.904002e+02,3.637035e+05,2.566077e-03,6.000000e+02}, - {3.241928e-03,7.024929e-03,2.212493e+07,8.600485e-03,3.963690e-01,4.834201e-01,3.736511e-03,5.724475e-03,4.790239e-03,2.766218e-03,1.151150e+00,1.150516e+00,2.089426e+02,8.666450e+02,1.898220e+02,2.862496e+02,3.056143e+05,7.039800e-03,6.000000e+02}, - {4.617594e-03,3.157739e-03,5.569465e+07,8.221076e-03,7.918279e-01,9.995014e-01,1.338309e-04,1.319707e-03,2.896082e-03,4.359171e-03,1.007827e+00,1.812954e+00,5.332209e+02,2.973599e+02,3.271466e+02,2.622351e+02,3.821569e+05,1.407429e-03,6.000000e+02} + {4.634940e-03,1.215335e-03,6.073270e+07,3.594486e-04,6.134229e-01,9.134229e-01,2.747871e-03,1.911238e-03,5.913313e-03,1.057645e-03,1.782748e+00,1.571392e+00,3.868229e+02,2.248689e+02,3.101180e+02,1.395063e+02,latvap+latice,5.494606e-03,6.000000e+02}, + {6.175320e-13,4.432407e-03,8.029967e+07,1.905151e-03,2.190099e-01,7.031070e-01,4.172977e-05,7.315360e-03,7.280063e-03,1.378543e-03,1.461443e+00,1.507382e+00,8.452377e+02,1.971876e+02,2.389249e+02,1.497752e+02,latvap+latice,5.107905e-03,6.000000e+02}, + {4.519798e-03,7.348916e-03,7.420725e+07,2.220971e-03,1.882608e-01,2.934182e-01,4.957590e-03,2.550256e-03,3.136926e-03,4.498115e-03,1.433526e+00,1.207516e+00,9.716844e+02,5.602546e+01,1.389465e+02,1.075863e+02,latvap+latice,6.771428e-03,6.000000e+02}, + {7.169182e-03,6.657331e-03,9.807967e+07,7.981196e-03,2.914473e-01,6.375719e-01,2.420032e-03,1.223012e-03,7.685516e-03,5.207024e-03,1.644865e+00,1.433872e+00,3.825069e+02,6.550300e+02,1.833466e+02,1.741918e+02,latvap+latice,3.792982e-03,6.000000e+02}, + {1.103118e-03,9.158125e-03,3.136196e+07,4.286154e-03,2.699078e-01,4.668103e-01,9.645460e-03,6.379119e-03,8.283285e-03,3.342400e-03,1.546698e+00,1.417916e+00,9.289270e+02,9.844129e+02,2.543202e+02,1.932996e+02,latvap+latice,2.693119e-03,6.000000e+02}, + {4.308000e-03,8.168535e-03,7.439969e+07,5.131497e-03,6.851225e-01,3.298025e-01,4.331812e-03,2.814373e-03,3.592807e-03,1.527499e-03,1.856943e+00,1.003269e+00,9.165690e+02,9.379921e+02,2.163204e+02,3.165814e+02,latvap+latice,6.801393e-03,6.000000e+02}, + {0.000000e-00,3.318968e-03,4.664041e+07,8.737282e-03,2.585907e-01,6.297295e-02,8.747418e-03,2.710437e-03,2.164895e-03,9.455725e-03,1.241506e+00,1.561393e+00,2.492674e+02,6.546182e+02,2.228772e+02,2.147968e+02,latvap+latice,5.903261e-03,6.000000e+02}, + {7.677170e-03,6.069057e-05,6.404241e+07,3.094233e-03,3.755403e-01,5.026876e-01,4.723817e-03,1.204228e-03,6.156526e-03,8.194797e-03,1.361509e+00,1.772751e+00,6.420537e+01,4.043364e+02,2.833110e+02,3.314521e+02,latvap+latice,2.996696e-03,6.000000e+02}, + + {9.999294e-03,3.138400e-03,2.355097e+07,9.897893e-03,7.667177e-01,9.739270e-01,4.221430e-03,3.570130e-03,8.370033e-03,9.527208e-03,1.597218e+00,1.111438e+00,7.832357e+02,8.364566e+02,2.854867e+02,2.340771e+02,latvap+latice,7.235757e-03,6.000000e+02}, + {8.841793e-03,3.530456e-03,9.618284e+07,9.311658e-03,3.458590e-01,6.978258e-01,1.279864e-03,4.652008e-03,1.869728e-03,8.931663e-03,1.712564e+00,1.223882e+00,9.692403e+02,2.358558e+02,3.204043e+02,1.827677e+02,latvap+latice,7.646405e-03,6.000000e+02}, + {1.425612e-03,6.653411e-04,2.843806e+07,1.922560e-03,9.100262e-01,0.996264e-01,8.973183e-04,9.857420e-03,6.221419e-03,8.133433e-03,1.815337e+00,1.885506e+00,5.508742e+02,1.612139e+02,2.798523e+02,2.631136e+02,latvap+latice,4.148666e-03,6.000000e+02}, + {4.125177e-04,4.056163e-03,2.716439e+07,6.484214e-03,1.658752e-01,2.859102e-01,5.724081e-03,6.282997e-03,7.313187e-03,6.049825e-03,1.140910e+00,1.145941e+00,7.490652e+02,5.011633e+02,1.986541e+02,2.745566e+02,latvap+latice,6.784784e-03,6.000000e+02}, + {5.010628e-03,2.863789e-04,8.953841e+07,3.953058e-03,1.135952e-01,9.718675e-01,1.846157e-03,5.743094e-03,2.842649e-03,8.155366e-03,1.227867e+00,1.894249e+00,1.161776e+02,3.578576e+02,1.240083e+02,1.639791e+02,latvap+latice,4.497257e-03,6.000000e+02}, + {9.487866e-03,6.584660e-03,6.149682e+06,9.413342e-03,4.757261e-01,6.503885e-01,1.078922e-03,3.489665e-03,3.059596e-03,9.285703e-03,1.192620e+00,1.967205e+00,5.085628e+02,3.741816e+01,1.196252e+02,2.904002e+02,latvap+latice,2.566077e-03,6.000000e+02}, + {3.241928e-03,7.024929e-03,2.212493e+07,8.600485e-03,3.963690e-01,4.834201e-01,3.736511e-03,5.724475e-03,4.790239e-03,2.766218e-03,1.151150e+00,1.150516e+00,2.089426e+02,8.666450e+02,1.898220e+02,2.862496e+02,latvap+latice,7.039800e-03,6.000000e+02}, + {4.617594e-03,3.157739e-03,5.569465e+07,8.221076e-03,7.918279e-01,9.995014e-01,1.338309e-04,1.319707e-03,2.896082e-03,4.359171e-03,1.007827e+00,1.812954e+00,5.332209e+02,2.973599e+02,3.271466e+02,2.622351e+02,latvap+latice,1.407429e-03,6.000000e+02} }; // Sync to device @@ -191,7 +192,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip // Init pack inputs Spack qr_incld,qc_incld,nr_incld,qi_incld, cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt; + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt; Scalar dt; @@ -215,7 +216,6 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip epsi_tot[s] = espd_device(vs).epsi_tot; t[s] = espd_device(vs).t; t_prev[s] = espd_device(vs).t_prev; - latent_heat_sublim[s]=espd_device(vs).latent_heat_sublim; dqsdt[s]=espd_device(vs).dqsdt; dt=espd_device(vs).dt; //qr2qv_evap_tend[s] = espd_device(vs).qr2qv_evap_tend; //PMC shouldn't have to init output vars. @@ -224,7 +224,7 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Functions::evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt, + ab,abi,epsr,epsi_tot,t,t_prev,dqsdt,dt, qr2qv_evap_tend,nr_evap_tend); // Copy results back into views diff --git a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp index 3ce02ced4597..5d7b4de4ea7e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp @@ -24,12 +24,12 @@ struct UnitWrap::UnitTest::TestLatentHeat { static void run_latent_heat_bfb() { + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + LatentHeatData latent_fortran[] = { // its, ite, kts, kte LatentHeatData(1, 7, 1, 10), - LatentHeatData(1, 7, 1, 10), - LatentHeatData(1, 7, 1, 10), - LatentHeatData(1, 7, 1, 10), }; static constexpr Int num_runs = sizeof(latent_fortran) / sizeof(LatentHeatData); @@ -45,16 +45,11 @@ struct UnitWrap::UnitTest::TestLatentHeat { LatentHeatData& h = latent_fortran[i]; get_latent_heat(h); - LatentHeatData& d = latent_cxx[i]; - get_latent_heat_f(d.its, d.ite, d.kts, d.kte, d.v, d.s, d.f); - if (SCREAM_BFB_TESTING) { - REQUIRE(h.total(h.v) == d.total(d.v)); - for (Int j = 0; j < h.total(h.v); ++j) { - REQUIRE(d.v[j] == h.v[j]); - REQUIRE(d.s[j] == h.s[j]); - REQUIRE(d.f[j] == h.f[j]); + REQUIRE(h.v[j] == latvap); + REQUIRE(h.s[j] == (latvap+latice)); + REQUIRE(h.f[j] == latice); } } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp index 78f7ab986c66..caf6638d5cd5 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp @@ -25,27 +25,30 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { { using KTH = KokkosTypes; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + IceWetGrowthData self[max_pack_size] = { // rho,temp,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,kap,mu,sc,qv,qc_incld,qi_incld,ni_incld,qr_incld,log_wetgrowth,qr2qi_collect_tend,qc2qi_collect_tend,qc_growth_rate,nr_ice_shed_tend,qc2qr_ice_shed_tend - {4.056000E-03, 1.023000E+02, 1.201000E+02, 9.002000E-04, 8.215000E-04, 8.852000E-01, 0.174000E+00, 1.221000E-14, 5.100000E-03, 9.558000E-04, 1.213000E-03, 9.653000E-04, 1.023000E-01, 4.098000E-02, 2.098000E-02, 9.952000E+03, 1.023000E-05, false, 1.241456E-04, 9.021345E-02, 1.043000E-01, 1.921000E-02, 0.242000E-02}, - {6.852000E-02, 1.120000E+02, 2.450000E+02, 9.321000E-04, 9.124000E-04, 8.852000E-01, 0.374000E+00, 1.221000E-13, 4.100000E-03, 9.558000E-04, 2.560000E-03, 1.764000E-03, 2.346000E-01, 5.632000E-02, 3.024000E-02, 9.952000E+03, 2.093000E-05, false, 2.341678E-04, 1.092432E-02, 2.903000E-01, 2.125000E-02, 0.342000E-02}, - {8.852000E-02, 1.210000E+02, 3.420000E+02, 9.623000E-04, 9.432000E-04, 8.900000E-01, 0.123000E+00, 1.221000E-12, 3.100000E-03, 9.558000E-04, 3.211000E-03, 3.421000E-03, 3.421000E-01, 6.542000E-02, 4.567000E-02, 9.952000E+03, 3.091000E-05, false, 3.215234E-04, 2.098987E-02, 3.450000E-01, 3.490000E-02, 0.932000E-02}, - {1.902000E-01, 1.326000E+02, 4.321000E+02, 9.982000E-04, 9.623000E-04, 9.900000E-01, 0.123000E+00, 1.221000E-11, 2.100000E-03, 9.558000E-04, 4.121000E-03, 4.569000E-03, 4.673000E-01, 7.902000E-02, 5.321000E-02, 9.952000E+03, 4.521000E-05, false, 4.675567E-04, 3.214982E-02, 4.290000E-01, 4.590000E-02, 1.025000E-01}, - - {2.201000E-01, 1.456000E+02, 5.670000E+02, 1.234000E-03, 9.723000E-04, 0.100000E+01, 0.174000E+00, 1.221000E-10, 1.100000E-03, 2.550008E-05, 4.980000E-03, 5.621000E-03, 5.420000E-01, 8.021000E-02, 6.902000E-02, 9.952000E+04, 5.678000E-05, false, 5.389236E-04, 4.125969E-02, 5.098000E-01, 5.921000E-02, 2.031000E-01}, - {3.502000E-01, 1.780009E+02, 6.832000E+02, 1.562000E-03, 1.024000E-03, 0.100000E+01, 0.374000E+00, 1.221000E-09, 8.100000E-04, 2.558000E-05, 5.643000E-03, 7.367000E-03, 6.782000E-01, 9.253000E-02, 8.045000E-02, 9.952000E+04, 6.902000E-05, false, 6.432654E-04, 5.389457E-02, 6.723000E-01, 6.093000E-02, 4.098000E-01}, - {4.852000E-01, 2.100009E+02, 7.090000E+02, 2.101000E-03, 1.235000E-03, 0.100000E+01, 0.123000E+00, 1.221000E-08, 4.100000E-04, 2.558000E-05, 7.892000E-03, 9.087000E-03, 8.213000E-01, 1.256000E-01, 9.134000E-02, 9.952000E+04, 8.367000E-05, false, 7.210983E-04, 6.476985E-02, 8.902000E-01, 8.345000E-02, 8.023000E-01}, - {5.852000E-01, 2.310000E+02, 9.215000E+02, 2.312000E-03, 1.456000E-03, 0.100000E+01, 0.123000E+00, 1.221000E-07, 2.100000E-04, 2.558000E-05, 9.321000E-03, 1.245000E-02, 1.067000E-00, 2.347000E-01, 1.092000E-01, 9.952000E+04, 9.098000E-05, false, 8.543367E-04, 8.213186E-02, 9.021000E-01, 9.321000E-02, 9.098000E-01}, - - {6.852000E-01, 2.563000E+02, 1.089000E+03, 3.601000E-03, 1.864000E-03, 0.950000E+00, 0.150000E+00, 1.221000E-06, 9.952000E-05, 4.596000E-05, 1.453000E-02, 2.543000E-02, 2.345000E-00, 3.578000E-01, 2.873000E-01, 1.734000E+04, 1.023000E-04, false, 9.021215E-04, 9.023367E-02, 1.023000E-00, 1.056000E-01, 1.256000E-00}, - {7.852000E-01, 2.789000E+02, 3.754000E+03, 3.891000E-03, 2.093000E-03, 0.950000E+00, 0.374000E+00, 1.221000E-05, 4.952000E-05, 4.596000E-05, 2.789000E-02, 4.367000E-02, 3.890000E-00, 4.980000E-01, 3.468000E-01, 1.734000E+04, 2.146000E-04, false, 1.043468E-05, 1.094854E-02, 2.012000E-00, 2.893000E-01, 2.903000E-00}, - {8.852000E-01, 3.123000E+02, 8.902000E+03, 4.872000E-03, 2.345000E-03, 0.950000E+00, 0.123000E+00, 1.221000E-04, 1.952000E-05, 4.596000E-05, 4.256000E-02, 6.324000E-02, 4.120000E-00, 6.321000E-01, 4.890000E-01, 1.734000E+04, 4.321000E-04, false, 2.341763E-05, 2.126247E-03, 3.120000E-00, 3.456000E-01, 3.912000E-00}, - {9.852000E-01, 4.981000E+02, 1.092000E+04, 5.210000E-03, 3.210000E-03, 0.950000E+00, 0.123000E+00, 1.221000E-03, 9.952000E-06, 4.596000E-05, 6.821000E-02, 8.789000E-02, 5.320000E-00, 7.982000E-01, 6.921000E-01, 1.734000E+04, 5.821000E-04, false, 3.901479E-05, 3.874763E-03, 5.902000E-00, 5.092000E-01, 4.821000E-00}, - - {1.002000E+01, 1.234000E+03, 2.125000E+04, 6.012000E-03, 5.902000E-03, 1.069000E+00, 0.174000E+00, 1.221000E-02, 6.952000E-06, 6.596000E-05, 8.472000E-02, 1.543000E-01, 6.012000E-00, 8.902000E-01, 9.210000E-01, 1.734000E+04, 6.921000E-04, false, 4.521923E-05, 4.592698E-03, 6.091000E-00, 6.743000E-01, 5.602000E-00}, - {1.152000E+01, 2.120000E+03, 4.568000E+04, 6.342000E-03, 9.210000E-03, 1.069000E+00, 0.374000E+00, 1.221000E-02, 3.952000E-06, 6.596000E-05, 1.098000E-01, 3.456000E-01, 7.241000E-00, 9.102000E-01, 1.002000E-00, 1.734000E+04, 7.901000E-04, false, 5.236542E-05, 5.678873E-03, 7.231000E-00, 8.321000E-01, 6.092000E-00}, - {1.252000E+01, 3.145000E+03, 8.213000E+04, 9.290000E-03, 1.034000E-02, 1.069000E+00, 0.123000E+00, 1.221000E-02, 1.952000E-06, 6.596000E-05, 2.340006E-01, 5.632000E-01, 8.452000E-00, 1.003000E-01, 2.145000E-00, 1.734000E+04, 9.212000E-04, false, 6.732276E-05, 7.321873E-03, 8.234000E-00, 9.023000E-01, 7.201000E-00}, - {1.352000E+01, 4.742000E+03, 1.014000E+05, 1.234000E-02, 1.456000E-02, 1.069000E+00, 0.123000E+00, 1.221000E-02, 9.952000E-07, 6.596000E-05, 4.123000E-01, 6.128000E-01, 9.076000E-00, 2.831000E-01, 3.902000E-00, 1.734000E+04, 1.023000E-03, false, 7.902887E-05, 9.032908E-03, 9.021000E-00, 1.092000E-01, 8.096000E-00} + {4.056000E-03, 1.023000E+02, 1.201000E+02, 9.002000E-04, 8.215000E-04, 8.852000E-01, latvap, latice, 5.100000E-03, 9.558000E-04, 1.213000E-03, 9.653000E-04, 1.023000E-01, 4.098000E-02, 2.098000E-02, 9.952000E+03, 1.023000E-05, false, 1.241456E-04, 9.021345E-02, 1.043000E-01, 1.921000E-02, 0.242000E-02}, + {6.852000E-02, 1.120000E+02, 2.450000E+02, 9.321000E-04, 9.124000E-04, 8.852000E-01, latvap, latice, 4.100000E-03, 9.558000E-04, 2.560000E-03, 1.764000E-03, 2.346000E-01, 5.632000E-02, 3.024000E-02, 9.952000E+03, 2.093000E-05, false, 2.341678E-04, 1.092432E-02, 2.903000E-01, 2.125000E-02, 0.342000E-02}, + {8.852000E-02, 1.210000E+02, 3.420000E+02, 9.623000E-04, 9.432000E-04, 8.900000E-01, latvap, latice, 3.100000E-03, 9.558000E-04, 3.211000E-03, 3.421000E-03, 3.421000E-01, 6.542000E-02, 4.567000E-02, 9.952000E+03, 3.091000E-05, false, 3.215234E-04, 2.098987E-02, 3.450000E-01, 3.490000E-02, 0.932000E-02}, + {1.902000E-01, 1.326000E+02, 4.321000E+02, 9.982000E-04, 9.623000E-04, 9.900000E-01, latvap, latice, 2.100000E-03, 9.558000E-04, 4.121000E-03, 4.569000E-03, 4.673000E-01, 7.902000E-02, 5.321000E-02, 9.952000E+03, 4.521000E-05, false, 4.675567E-04, 3.214982E-02, 4.290000E-01, 4.590000E-02, 1.025000E-01}, + + {2.201000E-01, 1.456000E+02, 5.670000E+02, 1.234000E-03, 9.723000E-04, 0.100000E+01, latvap, latice, 1.100000E-03, 2.550008E-05, 4.980000E-03, 5.621000E-03, 5.420000E-01, 8.021000E-02, 6.902000E-02, 9.952000E+04, 5.678000E-05, false, 5.389236E-04, 4.125969E-02, 5.098000E-01, 5.921000E-02, 2.031000E-01}, + {3.502000E-01, 1.780009E+02, 6.832000E+02, 1.562000E-03, 1.024000E-03, 0.100000E+01, latvap, latice, 8.100000E-04, 2.558000E-05, 5.643000E-03, 7.367000E-03, 6.782000E-01, 9.253000E-02, 8.045000E-02, 9.952000E+04, 6.902000E-05, false, 6.432654E-04, 5.389457E-02, 6.723000E-01, 6.093000E-02, 4.098000E-01}, + {4.852000E-01, 2.100009E+02, 7.090000E+02, 2.101000E-03, 1.235000E-03, 0.100000E+01, latvap, latice, 4.100000E-04, 2.558000E-05, 7.892000E-03, 9.087000E-03, 8.213000E-01, 1.256000E-01, 9.134000E-02, 9.952000E+04, 8.367000E-05, false, 7.210983E-04, 6.476985E-02, 8.902000E-01, 8.345000E-02, 8.023000E-01}, + {5.852000E-01, 2.310000E+02, 9.215000E+02, 2.312000E-03, 1.456000E-03, 0.100000E+01, latvap, latice, 2.100000E-04, 2.558000E-05, 9.321000E-03, 1.245000E-02, 1.067000E-00, 2.347000E-01, 1.092000E-01, 9.952000E+04, 9.098000E-05, false, 8.543367E-04, 8.213186E-02, 9.021000E-01, 9.321000E-02, 9.098000E-01}, + + {6.852000E-01, 2.563000E+02, 1.089000E+03, 3.601000E-03, 1.864000E-03, 0.950000E+00, latvap, latice, 9.952000E-05, 4.596000E-05, 1.453000E-02, 2.543000E-02, 2.345000E-00, 3.578000E-01, 2.873000E-01, 1.734000E+04, 1.023000E-04, false, 9.021215E-04, 9.023367E-02, 1.023000E-00, 1.056000E-01, 1.256000E-00}, + {7.852000E-01, 2.789000E+02, 3.754000E+03, 3.891000E-03, 2.093000E-03, 0.950000E+00, latvap, latice, 4.952000E-05, 4.596000E-05, 2.789000E-02, 4.367000E-02, 3.890000E-00, 4.980000E-01, 3.468000E-01, 1.734000E+04, 2.146000E-04, false, 1.043468E-05, 1.094854E-02, 2.012000E-00, 2.893000E-01, 2.903000E-00}, + {8.852000E-01, 3.123000E+02, 8.902000E+03, 4.872000E-03, 2.345000E-03, 0.950000E+00, latvap, latice, 1.952000E-05, 4.596000E-05, 4.256000E-02, 6.324000E-02, 4.120000E-00, 6.321000E-01, 4.890000E-01, 1.734000E+04, 4.321000E-04, false, 2.341763E-05, 2.126247E-03, 3.120000E-00, 3.456000E-01, 3.912000E-00}, + {9.852000E-01, 4.981000E+02, 1.092000E+04, 5.210000E-03, 3.210000E-03, 0.950000E+00, latvap, latice, 9.952000E-06, 4.596000E-05, 6.821000E-02, 8.789000E-02, 5.320000E-00, 7.982000E-01, 6.921000E-01, 1.734000E+04, 5.821000E-04, false, 3.901479E-05, 3.874763E-03, 5.902000E-00, 5.092000E-01, 4.821000E-00}, + + {1.002000E+01, 1.234000E+03, 2.125000E+04, 6.012000E-03, 5.902000E-03, 1.069000E+00, latvap, latice, 6.952000E-06, 6.596000E-05, 8.472000E-02, 1.543000E-01, 6.012000E-00, 8.902000E-01, 9.210000E-01, 1.734000E+04, 6.921000E-04, false, 4.521923E-05, 4.592698E-03, 6.091000E-00, 6.743000E-01, 5.602000E-00}, + {1.152000E+01, 2.120000E+03, 4.568000E+04, 6.342000E-03, 9.210000E-03, 1.069000E+00, latvap, latice, 3.952000E-06, 6.596000E-05, 1.098000E-01, 3.456000E-01, 7.241000E-00, 9.102000E-01, 1.002000E-00, 1.734000E+04, 7.901000E-04, false, 5.236542E-05, 5.678873E-03, 7.231000E-00, 8.321000E-01, 6.092000E-00}, + {1.252000E+01, 3.145000E+03, 8.213000E+04, 9.290000E-03, 1.034000E-02, 1.069000E+00, latvap, latice, 1.952000E-06, 6.596000E-05, 2.340006E-01, 5.632000E-01, 8.452000E-00, 1.003000E-01, 2.145000E-00, 1.734000E+04, 9.212000E-04, false, 6.732276E-05, 7.321873E-03, 8.234000E-00, 9.023000E-01, 7.201000E-00}, + {1.352000E+01, 4.742000E+03, 1.014000E+05, 1.234000E-02, 1.456000E-02, 1.069000E+00, latvap, latice, 9.952000E-07, 6.596000E-05, 4.123000E-01, 6.128000E-01, 9.076000E-00, 2.831000E-01, 3.902000E-00, 1.734000E+04, 1.023000E-03, false, 7.902887E-05, 9.032908E-03, 9.021000E-00, 1.092000E-01, 8.096000E-00} }; // Sync to device @@ -64,7 +67,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { const Int offset = i * Spack::n; // Init pack inputs - Spack rho,temp, pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,kap,mu,sc, + Spack rho,temp, pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,dv,kap,mu,sc, qv,qc_incld,qi_incld,ni_incld,qr_incld; Smask log_wetgrowth; @@ -78,8 +81,6 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { rhofaci[s] = self_device(vs).rhofaci; table_val_qi2qr_melting[s] = self_device(vs).table_val_qi2qr_melting; table_val_qi2qr_vent_melt[s] = self_device(vs).table_val_qi2qr_vent_melt; - latent_heat_vapor[s] = self_device(vs).latent_heat_vapor; - latent_heat_fusion[s] = self_device(vs).latent_heat_fusion; dv[s] = self_device(vs).dv; kap[s] = self_device(vs).kap; mu[s] = self_device(vs).mu; @@ -97,7 +98,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { log_wetgrowth.set(s, self_device(vs).log_wetgrowth); } - Functions::ice_cldliq_wet_growth(rho, temp, pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, kap, mu, sc, + Functions::ice_cldliq_wet_growth(rho, temp, pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, dv, kap, mu, sc, qv, qc_incld, qi_incld, ni_incld, qr_incld, log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend); diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp index 3e340f68710a..28c6582f8a14 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp @@ -23,29 +23,31 @@ struct UnitWrap::UnitTest::TestP3IceMelting { static void ice_melting_bfb(){ + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; // make array of input data (why not pass actual variables?). Copied 1st 4 rows 4x to fill pack size. IceMeltingData IceMelt[max_pack_size] = { //rho, T_atm, pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, sc, mu, kap, qv, qi_incld,ni_incld - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, - - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, - - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, - - {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, - {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, - {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,0.250E+07,0.334E+06,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, - {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,0.250E+07,0.334E+06,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05} + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, + + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, + + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05}, + + {0.117E+01,0.299E+03,0.101E+06,0.829E+00,0.122E+01,0.562E-01,latvap,latice,0.263E-04,0.601E+00,0.185E-04,0.261E-01,0.160E-01,0.510E-02, 0.195E-12}, + {0.114E+01,0.296E+03,0.973E+05,0.842E+00,0.122E+01,0.562E-01,latvap,latice,0.268E-04,0.601E+00,0.183E-04,0.259E-01,0.149E-01,0.510E-02, 0.195E-12}, + {0.977E+00,0.287E+03,0.809E+05,0.913E+00,0.122E+01,0.562E-01,latvap,latice,0.306E-04,0.599E+00,0.179E-04,0.253E-01,0.827E-02,0.000E+00, 0.000E+00}, + {0.103E+01,0.289E+03,0.862E+05,0.887E+00,0.636E-03,0.281E-04,latvap,latice,0.291E-04,0.600E+00,0.180E-04,0.254E-01,0.107E-01,0.510E-02, 0.336E+05} }; // Sync to device @@ -65,7 +67,7 @@ static void ice_melting_bfb(){ const Int offset = i * Spack::n; // Init pack inputs - Spack rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend; + Spack rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { rho[s] = IceMelt_device(vs).rho; T_atm[s] = IceMelt_device(vs).T_atm; @@ -73,8 +75,6 @@ static void ice_melting_bfb(){ rhofaci[s] = IceMelt_device(vs).rhofaci; table_val_qi2qr_melting[s] = IceMelt_device(vs).table_val_qi2qr_melting; table_val_qi2qr_vent_melt[s] = IceMelt_device(vs).table_val_qi2qr_vent_melt; - latent_heat_vapor[s] = IceMelt_device(vs).latent_heat_vapor; - latent_heat_fusion[s] = IceMelt_device(vs).latent_heat_fusion; dv[s] = IceMelt_device(vs).dv; sc[s] = IceMelt_device(vs).sc; mu[s] = IceMelt_device(vs).mu; @@ -86,7 +86,7 @@ static void ice_melting_bfb(){ ni2nr_melt_tend[s] = IceMelt_device(vs).ni2nr_melt_tend; } - Functions::ice_melting(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend); + Functions::ice_melting(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { IceMelt_device(vs).qi2qr_melt_tend = qi2qr_melt_tend[s]; diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index 02167261bf86..93c06a9054d7 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -187,6 +187,8 @@ static void run_bfb_ice_sed() static void run_bfb_homogeneous_freezing() { + constexpr Scalar latice = C::LatIce; + auto engine = setup_random_test(); HomogeneousFreezingData hfds_fortran[] = { @@ -203,6 +205,12 @@ static void run_bfb_homogeneous_freezing() for (auto& d : hfds_fortran) { const auto qsmall_r = std::make_pair(C::QSMALL/2, C::QSMALL*2); d.randomize(engine, { {d.T_atm, {C::T_homogfrz - 10, C::T_homogfrz + 10}}, {d.qc, qsmall_r}, {d.qr, qsmall_r} }); + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + for (int k=0; k::TestIceSupersatConservation { static void run_bfb() { + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; + auto engine = setup_random_test(); IceSupersatConservationData f90_data[max_pack_size]; @@ -27,6 +30,10 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation { for (auto& d : f90_data) { d.randomize(engine); d.dt = f90_data[0].dt; // hold this fixed, it is not packed data + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + d.latent_heat_sublim = latvap+latice; } // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that @@ -46,10 +53,9 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation { const Int offset = i * Spack::n; // Init pack inputs - Spack cld_frac_i, latent_heat_sublim, qidep, qinuc, qv, qv_sat_i, t_atm, qi2qv_sublim_tend, qr2qv_evap_tend; + Spack cld_frac_i, qidep, qinuc, qv, qv_sat_i, t_atm, qi2qv_sublim_tend, qr2qv_evap_tend; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { cld_frac_i[s] = cxx_device(vs).cld_frac_i; - latent_heat_sublim[s] = cxx_device(vs).latent_heat_sublim; qidep[s] = cxx_device(vs).qidep; qinuc[s] = cxx_device(vs).qinuc; qv[s] = cxx_device(vs).qv; @@ -59,7 +65,7 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation { qr2qv_evap_tend[s] = cxx_device(vs).qr2qv_evap_tend; } - Functions::ice_supersat_conservation(qidep, qinuc, cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, cxx_device(offset).dt, qi2qv_sublim_tend, qr2qv_evap_tend); + Functions::ice_supersat_conservation(qidep, qinuc, cld_frac_i, qv, qv_sat_i, t_atm, cxx_device(offset).dt, qi2qv_sublim_tend, qr2qv_evap_tend); // Copy spacks back into cxx_device view for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index bad0ca7014da..a804bd8756d5 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -57,6 +57,8 @@ static void run_bfb_p3_main_part1() constexpr Scalar T_zerodegc = C::T_zerodegc; constexpr Scalar sup_upper = -0.05; constexpr Scalar sup_lower = -0.1; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; P3MainPart1Data isds_fortran[] = { // kts, kte, ktop, kbot, kdir, do_predict_nc, do_prescribed_CCN, dt @@ -74,6 +76,14 @@ static void run_bfb_p3_main_part1() {d.T_atm, {T_zerodegc - 10, T_zerodegc + 10}}, {d.qv_supersat_i, {sup_lower -.05, sup_upper + .05}}, {d.qc, qsmall_r}, {d.qr, qsmall_r}, {d.qi, qsmall_r} }); + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + for (int k=0; k(); } diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index 55501a30aa33..a1b270562819 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -25,14 +25,14 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { using physics = scream::physics::Functions; constexpr Scalar inv_cp = C::INV_CP; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //Start with reasonable values //============================ Spack pres(100000); Spack t_atm(270); Spack qv(1.7e-3); - Spack latent_heat_vapor(2.5e6); - Spack latent_heat_sublim(2.838e6); Scalar dt=60; Spack qidep(0); Spack qinuc(0); //dep and nuc only used together, so only fiddle with one @@ -50,8 +50,8 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { Spack qv_sinks=qidep + qinuc; Spack qv_sources=qi2qv_sublim_tend_tmp + qr2qv_evap_tend_tmp; Spack qv_endstep=qv - qv_sinks*dt + qv_sources*dt; - Spack T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*latent_heat_sublim*inv_cp - - qr2qv_evap_tend_tmp*latent_heat_vapor*inv_cp )*dt; + Spack T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*(latvap+latice)*inv_cp + - qr2qv_evap_tend_tmp*latvap*inv_cp )*dt; Spack qsl = physics::qv_sat_dry(T_endstep,pres,false,context); //"false" means NOT sat w/ respect to ice //just require index 0 since all entries are identical @@ -59,7 +59,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { REQUIRE(qv_endstep[0]>qsl[0]); // inputs to this test are testing what we want. //Now update sublim and evap tends using prevent_liq_supersaturation - Functions::prevent_liq_supersaturation(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, + Functions::prevent_liq_supersaturation(pres, t_atm, qv, dt, qidep, qinuc, qi2qv_sublim_tend_tmp, qr2qv_evap_tend_tmp); //Finally, recompute liquid saturation @@ -67,8 +67,8 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { qv_sinks=qidep + qinuc; qv_sources=qi2qv_sublim_tend_tmp + qr2qv_evap_tend_tmp; qv_endstep=qv - qv_sinks*dt + qv_sources*dt; - T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*latent_heat_sublim*inv_cp - - qr2qv_evap_tend_tmp*latent_heat_vapor*inv_cp )*dt; + T_endstep=t_atm + ( (qv_sinks-qi2qv_sublim_tend_tmp)*(latvap+latice)*inv_cp + - qr2qv_evap_tend_tmp*latvap*inv_cp )*dt; qsl = physics::qv_sat_dry(T_endstep,pres,false,context); //"false" means NOT sat w/ respect to ice //just require index 0 since all entries are identical @@ -80,7 +80,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { Spack qi2qv_sublim_tend_tmp2(1e-4); Spack qr2qv_evap_tend_tmp2(1e-4); - Functions::prevent_liq_supersaturation(pres, t_atm, qv_tmp, latent_heat_vapor, latent_heat_sublim, + Functions::prevent_liq_supersaturation(pres, t_atm, qv_tmp, dt, qidep, qinuc, qi2qv_sublim_tend_tmp2, qr2qv_evap_tend_tmp2); //just require index 0 since all entries are identical. REQUIRE( qi2qv_sublim_tend_tmp2[0] ==0 ); @@ -90,6 +90,8 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { static void run_bfb() { + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; auto engine = setup_random_test(); @@ -100,6 +102,11 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { for (auto& d : f90_data) { d.randomize(engine); d.dt = f90_data[0].dt; // Hold this fixed, this is not packed data + + // C++ impl uses constants for latent_heat values. Manually set here + // so F90 can match + d.latent_heat_vapor = latvap; + d.latent_heat_sublim = latvap+latice; } // Create copies of data for use by cxx and sync it to device. Needs to happen before @@ -127,11 +134,9 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { // Init pack inputs Scalar dt; - Spack latent_heat_sublim, latent_heat_vapor, pres, qi2qv_sublim_tend, qidep, qinuc, qr2qv_evap_tend, qv, t_atm; + Spack pres, qi2qv_sublim_tend, qidep, qinuc, qr2qv_evap_tend, qv, t_atm; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { dt = cxx_device(vs).dt; //dt is scalar but PreventLiqSupersaturationData has diff val for each row. - latent_heat_sublim[s] = cxx_device(vs).latent_heat_sublim; - latent_heat_vapor[s] = cxx_device(vs).latent_heat_vapor; pres[s] = cxx_device(vs).pres; qi2qv_sublim_tend[s] = cxx_device(vs).qi2qv_sublim_tend; qidep[s] = cxx_device(vs).qidep; @@ -141,7 +146,7 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { t_atm[s] = cxx_device(vs).t_atm; } - Functions::prevent_liq_supersaturation(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend); + Functions::prevent_liq_supersaturation(pres, t_atm, qv, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend); // Copy spacks back into cxx_device view for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index df5b80cdc19e..3b994cfdad5d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -480,103 +480,105 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce constexpr Scalar nmltratio = C::nmltratio; constexpr Scalar dt = 1.8000E+03; constexpr bool do_predict_nc = true; + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //fortran generated data is input to the following P3UpdatePrognosticIceData pupidc[max_pack_size] = { {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, 3.4778E-04, 3.5801E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.1386E-07, 0.0000E+00, 0.0000E+00, 2.7053E-02, - 0.0000E+00, 1.9209E-10, 1.0686E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 1.9209E-10, 1.0686E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 4.5312E+02, 2.8720E+02, 5.0000E-03, 6.4286E-05, 1.2344E+08, 7.3684E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 6.4286E-05, 1.0000E-02}, {2.1097E-18, 2.7648E-09, 3.8261E-09, 3.7754E+06, 6.8685E-04, 0.0000E+00, 4.1018E-08, 5.1227E+04, 4.8876E-15, 1.3468E-03, 2.8059E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 7.1049E-07, 0.0000E+00, 0.0000E+00, 2.4547E-02, - 0.0000E+00, 2.8615E-10, 1.0741E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 2.8615E-10, 1.0741E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 3.4890E+02, 2.8642E+02, 5.0000E-03, 7.1429E-05, 1.2345E+08, 7.8947E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.1429E-05, 1.0000E-02}, {8.9820E-18, 4.2529E-09, 2.9520E-09, 3.7537E+06, 2.6598E-03, 0.0000E+00, 4.3700E-08, 5.1171E+04, 1.4266E-14, 5.2153E-03, 1.9880E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 9.0244E-07, 0.0000E+00, 0.0000E+00, 2.1083E-02, - 0.0000E+00, 3.7631E-10, 1.0796E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 3.7631E-10, 1.0796E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.8656E+02, 2.8565E+02, 5.0000E-03, 7.8571E-05, 1.2345E+08, 8.4211E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 7.8571E-05, 1.0000E-02}, {3.7942E-17, 6.0115E-09, 1.8004E-09, 3.7310E+06, 1.0300E-02, 0.0000E+00, 4.6119E-08, 5.1112E+04, 4.4518E-14, 2.0196E-02, 1.1226E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 1.0879E-06, 0.0000E+00, 0.0000E+00, 1.7646E-02, - 0.0000E+00, 4.5891E-10, 1.0853E+00, 3.3370E+05, 2.8347E+06, do_predict_nc, true, dt, nmltratio, + 0.0000E+00, 4.5891E-10, 1.0853E+00, latvap+latice, latice, do_predict_nc, true, dt, nmltratio, 2.4570E+02, 2.8489E+02, 5.0000E-03, 8.5714E-05, 1.2345E+08, 8.9474E-06, 1.0000E+06, 1.0000E-04, 1.0000E+06, 8.5714E-05, 1.0000E-02}, }; @@ -599,9 +601,9 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce const Int offset = i * Spack::n; // Init pack inputs - Spack qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend, ncshdc, qr2qi_collect_tend, nr_collect_tend, - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, - ni_nucleat_tend, ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, latent_heat_fusion, latent_heat_sublim, + Spack qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend, ncshdc, qr2qi_collect_tend, nr_collect_tend, + qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, + ni_nucleat_tend, ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, rho_qm_cloud, th_atm, qv, qc, nc, qr, nr, qi, ni, qm, bm; Scalar dt; bool do_predict_nc; @@ -633,8 +635,6 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce ni_sublim_tend[s] = pupidc_device(vs).ni_sublim_tend; qc2qi_berg_tend[s] = pupidc_device(vs).qc2qi_berg_tend; inv_exner[s] = pupidc_device(vs).inv_exner; - latent_heat_fusion[s] = pupidc_device(vs).latent_heat_fusion; - latent_heat_sublim[s] = pupidc_device(vs).latent_heat_sublim; rho_qm_cloud[s] = pupidc_device(vs).rho_qm_cloud; th_atm[s] = pupidc_device(vs).th_atm; @@ -654,7 +654,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce Functions::update_prognostic_ice(qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend,ncshdc, qr2qi_collect_tend, nr_collect_tend, qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, ni_nucleat_tend, - ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, latent_heat_sublim, latent_heat_fusion, + ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, do_predict_nc, log_wetgrowth, dt, pupidc_device(0).nmltratio, rho_qm_cloud, th_atm, qv, qi, ni, qm, bm, qc, nc, qr, nr); @@ -705,26 +705,28 @@ template struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables { static void get_time_space_phys_variables_unit_bfb_tests(){ + constexpr Scalar latvap = C::LatVap; + constexpr Scalar latice = C::LatIce; //fortran generated data is input to the following GetTimeSpacePhysVarsData gtspvd[max_pack_size] = { // T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i - {2.9792E+02, 9.8711E+04, 1.1532E+00, 2.5010E+06, 2.8347E+06, 2.0321E-02, 2.0321E-02}, - {2.9792E+02, 9.8711E+04, 1.1532E+00, 2.5010E+06, 2.8347E+06, 2.0321E-02, 2.0321E-02}, - {2.9583E+02, 9.7322E+04, 1.1449E+00, 2.5010E+06, 2.8347E+06, 1.8120E-02, 1.8120E-02}, - {2.9375E+02, 9.5933E+04, 1.1366E+00, 2.5010E+06, 2.8347E+06, 1.6134E-02, 1.6134E-02}, - {2.8959E+02, 9.3156E+04, 1.1196E+00, 2.5010E+06, 2.8347E+06, 1.2729E-02, 1.2729E-02}, - {2.8750E+02, 9.1767E+04, 1.1109E+00, 2.5010E+06, 2.8347E+06, 1.1279E-02, 1.1279E-02}, - {2.8542E+02, 9.0378E+04, 1.1020E+00, 2.5010E+06, 2.8347E+06, 9.9759E-03, 9.9759E-03}, - {2.8334E+02, 8.8989E+04, 1.0931E+00, 2.5010E+06, 2.8347E+06, 8.8076E-03, 8.8076E-03}, - {2.8125E+02, 8.7600E+04, 1.0840E+00, 2.5010E+06, 2.8347E+06, 7.7615E-03, 7.7615E-03}, - {2.7917E+02, 8.6211E+04, 1.0748E+00, 2.5010E+06, 2.8347E+06, 6.8265E-03, 6.8265E-03}, - {2.7709E+02, 8.4822E+04, 1.0654E+00, 2.5010E+06, 2.8347E+06, 5.9921E-03, 5.9921E-03}, - {2.7501E+02, 8.3433E+04, 1.0559E+00, 2.5010E+06, 2.8347E+06, 5.2488E-03, 5.2488E-03}, - {2.7292E+02, 8.2044E+04, 1.0463E+00, 2.5010E+06, 2.8347E+06, 4.5879E-03, 4.5766E-03}, - {2.7084E+02, 8.0656E+04, 1.0365E+00, 2.5010E+06, 2.8347E+06, 4.0015E-03, 3.9112E-03}, - {2.6876E+02, 7.9267E+04, 1.0265E+00, 2.5010E+06, 2.8347E+06, 3.4821E-03, 3.3349E-03}, - {2.6667E+02, 7.7878E+04, 1.0164E+00, 2.5010E+06, 2.8347E+06, 3.0231E-03, 2.8368E-03}, + {2.9792E+02, 9.8711E+04, 1.1532E+00, latvap, latvap+latice, 2.0321E-02, 2.0321E-02}, + {2.9792E+02, 9.8711E+04, 1.1532E+00, latvap, latvap+latice, 2.0321E-02, 2.0321E-02}, + {2.9583E+02, 9.7322E+04, 1.1449E+00, latvap, latvap+latice, 1.8120E-02, 1.8120E-02}, + {2.9375E+02, 9.5933E+04, 1.1366E+00, latvap, latvap+latice, 1.6134E-02, 1.6134E-02}, + {2.8959E+02, 9.3156E+04, 1.1196E+00, latvap, latvap+latice, 1.2729E-02, 1.2729E-02}, + {2.8750E+02, 9.1767E+04, 1.1109E+00, latvap, latvap+latice, 1.1279E-02, 1.1279E-02}, + {2.8542E+02, 9.0378E+04, 1.1020E+00, latvap, latvap+latice, 9.9759E-03, 9.9759E-03}, + {2.8334E+02, 8.8989E+04, 1.0931E+00, latvap, latvap+latice, 8.8076E-03, 8.8076E-03}, + {2.8125E+02, 8.7600E+04, 1.0840E+00, latvap, latvap+latice, 7.7615E-03, 7.7615E-03}, + {2.7917E+02, 8.6211E+04, 1.0748E+00, latvap, latvap+latice, 6.8265E-03, 6.8265E-03}, + {2.7709E+02, 8.4822E+04, 1.0654E+00, latvap, latvap+latice, 5.9921E-03, 5.9921E-03}, + {2.7501E+02, 8.3433E+04, 1.0559E+00, latvap, latvap+latice, 5.2488E-03, 5.2488E-03}, + {2.7292E+02, 8.2044E+04, 1.0463E+00, latvap, latvap+latice, 4.5879E-03, 4.5766E-03}, + {2.7084E+02, 8.0656E+04, 1.0365E+00, latvap, latvap+latice, 4.0015E-03, 3.9112E-03}, + {2.6876E+02, 7.9267E+04, 1.0265E+00, latvap, latvap+latice, 3.4821E-03, 3.3349E-03}, + {2.6667E+02, 7.7878E+04, 1.0164E+00, latvap, latvap+latice, 3.0231E-03, 2.8368E-03}, }; // Sync to device @@ -745,14 +747,12 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables const Int offset = i * Spack::n; // Init pack inputs - Spack T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii; + Spack T_atm, pres, rho, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii; for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { T_atm[s] = gtspvd_device(vs).T_atm; pres[s] = gtspvd_device(vs).pres; rho[s] = gtspvd_device(vs).rho; - latent_heat_vapor[s] = gtspvd_device(vs).latent_heat_vapor; - latent_heat_sublim[s] = gtspvd_device(vs).latent_heat_sublim; qv_sat_l[s] = gtspvd_device(vs).qv_sat_l; qv_sat_i[s] = gtspvd_device(vs).qv_sat_i; @@ -767,7 +767,7 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables eii[s] = gtspvd_device(vs).eii; } - Functions::get_time_space_phys_variables(T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, + Functions::get_time_space_phys_variables(T_atm, pres, rho, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii); // Copy results back into views @@ -775,8 +775,6 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables gtspvd_device(vs).T_atm = T_atm[s]; gtspvd_device(vs).pres = pres[s]; gtspvd_device(vs).rho = rho[s]; - gtspvd_device(vs).latent_heat_vapor = latent_heat_vapor[s]; - gtspvd_device(vs).latent_heat_sublim = latent_heat_sublim[s]; gtspvd_device(vs).qv_sat_l = qv_sat_l[s]; gtspvd_device(vs).qv_sat_i = qv_sat_i[s]; @@ -820,72 +818,73 @@ template struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq { static void update_prognostic_liquid_unit_bfb_tests(){ + constexpr Scalar latvap = C::LatVap; //fortran generated data is input to the following P3UpdatePrognosticLiqData pupldc[max_pack_size] = { {1.0631E-12, 1.0631E+00, 1.5833E-12, 1.5833E+00, 2.4190E-02, 0.0000E+00, 0.0000E+00, 0.0000E+00, 4.2517E+00, - true , true , 8.6718E-01, 1.0037E+00, 2.5010E+06, 1.8000E+03, 2.9902E+02, 5.0000E-02, 1.0000E-06, 1.0000E+06, 1.0010E-06, + true , true , 8.6718E-01, 1.0037E+00, latvap, 1.8000E+03, 2.9902E+02, 5.0000E-02, 1.0000E-06, 1.0000E+06, 1.0010E-06, 6.3726E+05}, {3.2784E-08, 1.8780E+07, 2.1753E-11, 1.2461E+04, 7.8657E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.8748E+04, - true , true , 9.8387E-01, 1.0741E+00, 2.5010E+06, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, + true , true , 9.8387E-01, 1.0741E+00, latvap, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, -8.6159E+07}, {3.2796E-09, 1.8778E+07, 1.8830E-12, 1.0782E+04, 6.8061E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.3698E+04, - true , true , 9.0740E-01, 1.0293E+00, 2.5010E+06, 1.8000E+03, 2.9376E+02, 5.0000E-03, 5.9067E-06,-6.9543E+09, 1.0439E-04, + true , true , 9.0740E-01, 1.0293E+00, latvap, 1.8000E+03, 2.9376E+02, 5.0000E-03, 5.9067E-06,-6.9543E+09, 1.0439E-04, -1.6967E+07}, {6.5634E-09, 1.8778E+07, 3.8238E-12, 1.0940E+04, 6.9061E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.3181E+04, - true , true , 9.1484E-01, 1.0339E+00, 2.5010E+06, 1.8000E+03, 2.9291E+02, 5.0000E-03, 1.1821E-05,-6.9282E+09, 1.0615E-04, + true , true , 9.1484E-01, 1.0339E+00, latvap, 1.8000E+03, 2.9291E+02, 5.0000E-03, 1.1821E-05,-6.9282E+09, 1.0615E-04, -2.8223E+07}, {9.8516E-09, 1.8779E+07, 5.8258E-12, 1.1105E+04, 7.0101E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.2655E+04, - true , true , 9.2251E-01, 1.0386E+00, 2.5010E+06, 1.8000E+03, 2.9206E+02, 5.0000E-03, 1.7743E-05,-6.9009E+09, 1.0790E-04, + true , true , 9.2251E-01, 1.0386E+00, latvap, 1.8000E+03, 2.9206E+02, 5.0000E-03, 1.7743E-05,-6.9009E+09, 1.0790E-04, -3.9628E+07}, {1.3145E-08, 1.8779E+07, 7.8929E-12, 1.1276E+04, 7.1180E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.2122E+04, - true , true , 9.3043E-01, 1.0433E+00, 2.5010E+06, 1.8000E+03, 2.9123E+02, 5.0000E-03, 2.3674E-05,-6.8725E+09, 1.0963E-04, + true , true , 9.3043E-01, 1.0433E+00, latvap, 1.8000E+03, 2.9123E+02, 5.0000E-03, 2.3674E-05,-6.8725E+09, 1.0963E-04, -5.1189E+07}, {1.6443E-08, 1.8779E+07, 1.0029E-11, 1.1454E+04, 7.2303E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.1581E+04, - true , true , 9.3860E-01, 1.0482E+00, 2.5010E+06, 1.8000E+03, 2.9040E+02, 5.0000E-03, 2.9615E-05,-6.8428E+09, 1.1136E-04, + true , true , 9.3860E-01, 1.0482E+00, latvap, 1.8000E+03, 2.9040E+02, 5.0000E-03, 2.9615E-05,-6.8428E+09, 1.1136E-04, -6.2915E+07}, {1.9746E-08, 1.8779E+07, 1.2238E-11, 1.1639E+04, 7.3471E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.1031E+04, - true , true , 9.4705E-01, 1.0531E+00, 2.5010E+06, 1.8000E+03, 2.8958E+02, 5.0000E-03, 3.5565E-05,-6.8117E+09, 1.1308E-04, + true , true , 9.4705E-01, 1.0531E+00, latvap, 1.8000E+03, 2.8958E+02, 5.0000E-03, 3.5565E-05,-6.8117E+09, 1.1308E-04, -7.4813E+07}, {2.3047E-08, 1.8779E+07, 1.4521E-11, 1.1832E+04, 7.4688E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 6.0474E+04, - true , true , 9.5579E-01, 1.0582E+00, 2.5010E+06, 1.8000E+03, 2.8941E+02, 4.7949E-03, 4.1510E-05,-6.7792E+09, 1.4787E-05, + true , true , 9.5579E-01, 1.0582E+00, latvap, 1.8000E+03, 2.8941E+02, 4.7949E-03, 4.1510E-05,-6.7792E+09, 1.4787E-05, -8.2885E+07}, {2.6289E-08, 1.8779E+07, 1.6845E-11, 1.2033E+04, 7.5955E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.9907E+04, - true , true , 9.6483E-01, 1.0634E+00, 2.5010E+06, 1.8000E+03, 2.8972E+02, 4.4341E-03, 4.7350E-05,-6.7452E+09,-4.7350E-05, + true , true , 9.6483E-01, 1.0634E+00, latvap, 1.8000E+03, 2.8972E+02, 4.4341E-03, 4.7350E-05,-6.7452E+09,-4.7350E-05, -8.3634E+07}, {2.9533E-08, 1.8779E+07, 1.9253E-11, 1.2242E+04, 7.7277E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.9332E+04, - true , false , 9.7418E-01, 1.0686E+00, 2.5010E+06, 1.8000E+03, 2.9002E+02, 4.0751E-03, 5.3194E-05,-6.7096E+09,-5.3194E-05, + true , false , 9.7418E-01, 1.0686E+00, latvap, 1.8000E+03, 2.9002E+02, 4.0751E-03, 5.3194E-05,-6.7096E+09,-5.3194E-05, -8.4862E+07}, {3.2784E-08, 1.8780E+07, 2.1753E-11, 1.2461E+04, 7.8657E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.8748E+04, - true , false , 9.8387E-01, 1.0741E+00, 2.5010E+06, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, + true , false , 9.8387E-01, 1.0741E+00, latvap, 1.8000E+03, 2.9033E+02, 3.7211E-03, 5.9050E-05,-6.6723E+09,-5.9050E-05, -8.6159E+07}, {3.6045E-08, 1.8780E+07, 2.4356E-11, 1.2689E+04, 8.0098E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.8154E+04, - true , false , 9.9391E-01, 1.0796E+00, 2.5010E+06, 1.8000E+03, 2.9063E+02, 3.3756E-03, 6.4925E-05,-6.6333E+09,-6.4925E-05, + true , false , 9.9391E-01, 1.0796E+00, latvap, 1.8000E+03, 2.9063E+02, 3.3756E-03, 6.4925E-05,-6.6333E+09,-6.4925E-05, -8.7530E+07}, {3.9321E-08, 1.8780E+07, 2.7069E-11, 1.2928E+04, 8.1605E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.7552E+04, - true , false , 1.0043E+00, 1.0853E+00, 2.5010E+06, 1.8000E+03, 2.9092E+02, 3.0417E-03, 7.0827E-05,-6.5924E+09,-7.0827E-05, + true , false , 1.0043E+00, 1.0853E+00, latvap, 1.8000E+03, 2.9092E+02, 3.0417E-03, 7.0827E-05,-6.5924E+09,-7.0827E-05, -8.8982E+07}, {4.2614E-08, 1.8780E+07, 2.9903E-11, 1.3178E+04, 8.3182E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.6939E+04, - true , false , 1.0151E+00, 1.0911E+00, 2.5010E+06, 1.8000E+03, 2.9119E+02, 2.7224E-03, 7.6760E-05,-6.5494E+09,-7.6760E-05, + true , false , 1.0151E+00, 1.0911E+00, latvap, 1.8000E+03, 2.9119E+02, 2.7224E-03, 7.6760E-05,-6.5494E+09,-7.6760E-05, -9.0523E+07}, {4.5927E-08, 1.8780E+07, 3.2867E-11, 1.3440E+04, 8.4833E+03, 0.0000E+00, 0.0000E+00, 0.0000E+00, 5.6317E+04, - true , false , 1.0263E+00, 1.0970E+00, 2.5010E+06, 1.8000E+03, 2.9143E+02, 2.4202E-03, 8.2728E-05,-6.5044E+09,-8.2728E-05, + true , false , 1.0263E+00, 1.0970E+00, latvap, 1.8000E+03, 2.9143E+02, 2.4202E-03, 8.2728E-05,-6.5044E+09,-8.2728E-05, -9.0778E+07}, }; @@ -908,7 +907,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq // Init pack inputs Spack qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, inv_rho, - inv_exner, latent_heat_vapor, th_atm, qv, qc, nc, qr, nr; + inv_exner, th_atm, qv, qc, nc, qr, nr; bool do_predict_nc, do_prescribed_CCN; Scalar dt; @@ -929,7 +928,6 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq nr_selfcollect_tend[s] = pupldc_device(vs).nr_selfcollect_tend; inv_rho[s] = pupldc_device(vs).inv_rho; inv_exner[s] = pupldc_device(vs).inv_exner; - latent_heat_vapor[s] = pupldc_device(vs).latent_heat_vapor; th_atm[s] = pupldc_device(vs).th_atm; qv[s] = pupldc_device(vs).qv; @@ -941,7 +939,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq Functions::update_prognostic_liquid(qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, do_predict_nc, do_prescribed_CCN, inv_rho, inv_exner, - latent_heat_vapor, dt, th_atm, qv, qc, nc, qr, nr); + dt, th_atm, qv, qc, nc, qr, nr); // Copy results back into views pupldc_device(0).dt = dt; @@ -959,7 +957,6 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq pupldc_device(vs).nr_selfcollect_tend = nr_selfcollect_tend[s]; pupldc_device(vs).inv_rho = inv_rho[s]; pupldc_device(vs).inv_exner = inv_exner[s]; - pupldc_device(vs).latent_heat_vapor = latent_heat_vapor[s]; pupldc_device(vs).th_atm = th_atm[s]; pupldc_device(vs).qv = qv[s]; diff --git a/components/eamxx/src/physics/register_physics.hpp b/components/eamxx/src/physics/register_physics.hpp index 36afabf8de97..d837e96311f4 100644 --- a/components/eamxx/src/physics/register_physics.hpp +++ b/components/eamxx/src/physics/register_physics.hpp @@ -26,7 +26,11 @@ #ifdef EAMXX_HAS_MAM #include "physics/mam/eamxx_mam_microphysics_process_interface.hpp" #include "physics/mam/eamxx_mam_optics_process_interface.hpp" +#include "physics/mam/eamxx_mam_dry_deposition_process_interface.hpp" #include "physics/mam/eamxx_mam_aci_process_interface.hpp" +#include "physics/mam/eamxx_mam_wetscav_process_interface.hpp" +#include "physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp" +#include "physics/mam/eamxx_mam_constituent_fluxes_interface.hpp" #endif #ifdef EAMXX_HAS_COSP #include "physics/cosp/eamxx_cosp.hpp" @@ -63,7 +67,11 @@ inline void register_physics () { #ifdef EAMXX_HAS_MAM proc_factory.register_product("mam4_micro",&create_atmosphere_process); proc_factory.register_product("mam4_optics",&create_atmosphere_process); + proc_factory.register_product("mam4_drydep",&create_atmosphere_process); proc_factory.register_product("mam4_aci",&create_atmosphere_process); + proc_factory.register_product("mam4_wetscav",&create_atmosphere_process); + proc_factory.register_product("mam4_srf_online_emiss",&create_atmosphere_process); + proc_factory.register_product("mam4_constituent_fluxes",&create_atmosphere_process); #endif #ifdef EAMXX_HAS_COSP proc_factory.register_product("Cosp",&create_atmosphere_process); diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index fcb819f599a8..a8e47384552a 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -109,7 +109,6 @@ set(EAM_RRTMGP_DIR ${SCREAM_BASE_DIR}/../eam/src/physics/rrtmgp) # NOTE: The external RRTMGP build needs some fixes to work with CUDA in a library build, so for now we will build these ourselves set(EXTERNAL_SRC ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/kernels/mo_gas_optics_kernels.cpp - ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_constants.cpp ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/mo_rrtmgp_util_reorder.cpp ${EAM_RRTMGP_DIR}/external/cpp/rte/expand_and_transpose.cpp ${EAM_RRTMGP_DIR}/external/cpp/rte/kernels/mo_fluxes_broadband_kernels.cpp @@ -128,6 +127,9 @@ yakl_process_target(rrtmgp) # NOTE: cannot use 'PUBLIC' in target_link_libraries, # since yakl_process_target already used it # with the "plain" signature +if (NOT TARGET Kokkos::kokkos) + find_package(Kokkos REQUIRED) +endif () target_link_libraries(rrtmgp yakl Kokkos::kokkos) target_include_directories(rrtmgp PUBLIC ${SCREAM_BASE_DIR}/../../externals/YAKL diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 1ff3f27b2fce..75082e789ed6 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -23,6 +23,48 @@ using KT = KokkosTypes; using ExeSpace = KT::ExeSpace; using MemberType = KT::MemberType; +namespace { + +struct ConvertToRrtmgpSubview +{ + int beg; + int ncol; + + template + View subview1d(const View& v) const { + return View(v, std::make_pair(beg, beg+ncol)); + } + + template + View subview2d_impl(const View& v, const int inner_dim) const { + return View(v, std::make_pair(beg, beg+ncol), std::make_pair(0, inner_dim)); + } + +#ifdef RRTMGP_LAYOUT_LEFT + template + BufferView subview2d(const FieldView&, const BufferView& buffer_view, const int inner_dim) const { + return BufferView(buffer_view, std::make_pair(0, ncol), Kokkos::ALL); + } +#else + // Be sure to trim the excess + // items from the field manager views due to simd packs. If we don't trim, then + // check_range_k will fail due to looking at unused values. Once rrtmgp can handle + // packs, this won't be necessary. + template + FieldView subview2d(const FieldView& field_view, const BufferView&, const int inner_dim) const { + return subview2d_impl(field_view, inner_dim); + } +#endif + + template + View subview3d(const View& v) const { + // The range assumes these are buffer views, not fields + return View(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); + } +}; + +} + RRTMGPRadiation:: RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params) @@ -59,12 +101,15 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ if (m_iop) { // For IOP runs, we need to use the lat/lon from the - // IOP files instead of the geometry data. + // IOP files instead of the geometry data. Deep copy + // to device and sync to host since both will be used. m_lat = m_grid->get_geometry_data("lat").clone(); m_lat.deep_copy(m_iop->get_params().get("target_latitude")); + m_lat.sync_to_host(); m_lon = m_grid->get_geometry_data("lon").clone(); m_lon.deep_copy(m_iop->get_params().get("target_longitude")); + m_lon.sync_to_host(); } else { m_lat = m_grid->get_geometry_data("lat"); m_lon = m_grid->get_geometry_data("lon"); @@ -255,7 +300,12 @@ size_t RRTMGPRadiation::requested_buffer_size_in_bytes() const Buffer::num_3d_nlay_nswbands*m_col_chunk_size*(m_nlay)*m_nswbands + Buffer::num_3d_nlay_nlwbands*m_col_chunk_size*(m_nlay)*m_nlwbands + Buffer::num_3d_nlay_nswgpts*m_col_chunk_size*(m_nlay)*m_nswgpts + - Buffer::num_3d_nlay_nlwgpts*m_col_chunk_size*(m_nlay)*m_nlwgpts; + Buffer::num_3d_nlay_nlwgpts*m_col_chunk_size*(m_nlay)*m_nlwgpts * +#if defined(RRTMGP_ENABLE_YAKL) && defined(RRTMGP_ENABLE_KOKKOS) + 2; +#else + 1; +#endif return interface_request * sizeof(Real); } // RRTMGPRadiation::requested_buffer_size @@ -409,13 +459,7 @@ void RRTMGPRadiation::init_buffers(const ATMBufferManager &buffer_manager) mem += m_buffer.cld_tau_lw_bnd.totElems(); #endif - // During the transition to kokkos, the buffer views/arrays will point to the same memory, - // so care is needed to avoid repeating calculations in such a way that answers change. - // Example: buff_view(x) += foo; - // Stuff like this cannot be done twice when both kokkos and yakl are enabled #ifdef RRTMGP_ENABLE_KOKKOS - mem = reinterpret_cast(buffer_manager.get_memory()); - // 1d arrays m_buffer.mu0_k = decltype(m_buffer.mu0_k)(mem, m_col_chunk_size); mem += m_buffer.mu0_k.size(); @@ -596,7 +640,7 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { // Names of active gases auto gas_names_yakl_offset = string1dv(m_ngas); - m_gas_mol_weights = view_1d_real("gas_mol_weights",m_ngas); + m_gas_mol_weights = real1dk("gas_mol_weights",m_ngas); // the lookup function for getting the gas mol weights doesn't work on device auto gas_mol_w_host = Kokkos::create_mirror_view(m_gas_mol_weights); for (int igas = 0; igas < m_ngas; igas++) { @@ -624,17 +668,17 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.init(gas_names_yakl_offset,m_col_chunk_size,m_nlay); - rrtmgp::rrtmgp_initialize( + interface_t::rrtmgp_initialize( m_gas_concs_k, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, m_atm_logger ); VALIDATE_KOKKOS(m_gas_concs, m_gas_concs_k); - VALIDATE_KOKKOS(rrtmgp::k_dist_sw, rrtmgp::k_dist_sw_k); - VALIDATE_KOKKOS(rrtmgp::k_dist_lw, rrtmgp::k_dist_lw_k); - VALIDATE_KOKKOS(rrtmgp::cloud_optics_sw, rrtmgp::cloud_optics_sw_k); - VALIDATE_KOKKOS(rrtmgp::cloud_optics_lw, rrtmgp::cloud_optics_lw_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_sw, interface_t::k_dist_sw_k); + VALIDATE_KOKKOS(rrtmgp::k_dist_lw, interface_t::k_dist_lw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_sw, interface_t::cloud_optics_sw_k); + VALIDATE_KOKKOS(rrtmgp::cloud_optics_lw, interface_t::cloud_optics_lw_k); #endif // Set property checks for fields in this process @@ -847,12 +891,14 @@ void RRTMGPRadiation::run_impl (const double dt) { this->log(LogLevel::debug, "[RRTMGP::run_impl] Col chunk beg,end: " + std::to_string(beg) + ", " + std::to_string(beg+ncol) + "\n"); - + // d_tint and d_dz are used in eamxx calls and therefore + // must be layout right + ulrreal2dk d_tint = ulrreal2dk(m_buffer.d_tint.data(), m_col_chunk_size, m_nlay+1); + ulrreal2dk d_dz = ulrreal2dk(m_buffer.d_dz.data(), m_col_chunk_size, m_nlay); + auto d_mu0 = m_buffer.cosine_zenith; +#ifdef RRTMGP_ENABLE_YAKL // Create YAKL arrays. RRTMGP expects YAKL arrays with styleFortran, i.e., data has ncol // as the fastest index. For this reason we must copy the data. - // JGF: this doesn't appear to be copying the data, just returning a new array - // pointing to the same memory. -#ifdef RRTMGP_ENABLE_YAKL auto subview_1d = [&](const real1d v) -> real1d { return real1d(v.label(),v.myData,ncol); }; @@ -922,100 +968,68 @@ void RRTMGPRadiation::run_impl (const double dt) { auto cld_tau_lw_gpt = subview_3d(m_buffer.cld_tau_lw_gpt); #endif #ifdef RRTMGP_ENABLE_KOKKOS - // If YAKL is on, we don't want aliased memory in both the yakl and kokos - // subviews. - auto subview_1dk = [&](const real1dk v) -> real1dk { - real1dk subv(v, std::make_pair(0, ncol)); -#ifdef RRTMGP_ENABLE_YAKL - real1dk rv(v.label(), ncol); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; - auto subview_2dk = [&](const real2dk v) -> real2dk { - real2dk subv(v, std::make_pair(0, ncol), Kokkos::ALL); -#ifdef RRTMGP_ENABLE_YAKL - real2dk rv(v.label(), ncol, v.extent(1)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; -#endif - }; - auto subview_3dk = [&](const real3dk v) -> real3dk { - real3dk subv(v, std::make_pair(0, ncol), Kokkos::ALL, Kokkos::ALL); -#ifdef RRTMGP_ENABLE_YAKL - real3dk rv(v.label(), ncol, v.extent(1), v.extent(2)); - Kokkos::deep_copy(rv, subv); - return rv; -#else - return subv; + ConvertToRrtmgpSubview conv = {beg, ncol}; + + // Note, ncol will not necessary be m_col_chunk_size because the number of cols + // will not always be evenly divided by m_col_chunk_size. In most cases, the + // extra space will not cause any problems, but it does sometimes. + auto p_lay_k = conv.subview2d(d_pmid, m_buffer.p_lay_k, m_nlay); + auto t_lay_k = conv.subview2d(d_tmid, m_buffer.t_lay_k, m_nlay); + auto p_lev_k = conv.subview2d(d_pint, m_buffer.p_lev_k, m_nlay+1); + auto z_del_k = conv.subview2d(d_dz, m_buffer.z_del_k, m_nlay); + auto p_del_k = conv.subview2d(d_pdel, m_buffer.p_del_k, m_nlay); + auto t_lev_k = conv.subview2d(d_tint, m_buffer.t_lev_k, m_nlay+1); + auto sfc_alb_dir_k = m_buffer.sfc_alb_dir_k; + auto sfc_alb_dif_k = m_buffer.sfc_alb_dif_k; + auto sfc_alb_dir_vis_k = conv.subview1d(d_sfc_alb_dir_vis); + auto sfc_alb_dir_nir_k = conv.subview1d(d_sfc_alb_dir_nir); + auto sfc_alb_dif_vis_k = conv.subview1d(d_sfc_alb_dif_vis); + auto sfc_alb_dif_nir_k = conv.subview1d(d_sfc_alb_dif_nir); + auto qc_k = conv.subview2d(d_qc, m_buffer.qc_k, m_nlay); + auto nc_k = conv.subview2d(d_nc, m_buffer.nc_k, m_nlay); + auto qi_k = conv.subview2d(d_qi, m_buffer.qi_k, m_nlay); + auto cldfrac_tot_k = m_buffer.cldfrac_tot_k; + auto rel_k = conv.subview2d(d_rel, m_buffer.eff_radius_qc_k, m_nlay); + auto rei_k = conv.subview2d(d_rei, m_buffer.eff_radius_qi_k, m_nlay); + auto sw_flux_up_k = conv.subview2d(d_sw_flux_up, m_buffer.sw_flux_up_k, m_nlay+1); + auto sw_flux_dn_k = conv.subview2d(d_sw_flux_dn, m_buffer.sw_flux_dn_k, m_nlay+1); + auto sw_flux_dn_dir_k = conv.subview2d(d_sw_flux_dn_dir, m_buffer.sw_flux_dn_dir_k, m_nlay+1); + auto lw_flux_up_k = conv.subview2d(d_lw_flux_up, m_buffer.lw_flux_up_k, m_nlay+1); + auto lw_flux_dn_k = conv.subview2d(d_lw_flux_dn, m_buffer.lw_flux_dn_k, m_nlay+1); + auto sw_clnclrsky_flux_up_k = conv.subview2d(d_sw_clnclrsky_flux_up, m_buffer.sw_clnclrsky_flux_up_k, m_nlay+1); + auto sw_clnclrsky_flux_dn_k = conv.subview2d(d_sw_clnclrsky_flux_dn, m_buffer.sw_clnclrsky_flux_dn_k, m_nlay+1); + auto sw_clnclrsky_flux_dn_dir_k = conv.subview2d(d_sw_clnclrsky_flux_dn_dir, m_buffer.sw_clnclrsky_flux_dn_dir_k, m_nlay+1); + auto sw_clrsky_flux_up_k = conv.subview2d(d_sw_clrsky_flux_up, m_buffer.sw_clrsky_flux_up_k, m_nlay+1); + auto sw_clrsky_flux_dn_k = conv.subview2d(d_sw_clrsky_flux_dn, m_buffer.sw_clrsky_flux_dn_k, m_nlay+1); + auto sw_clrsky_flux_dn_dir_k = conv.subview2d(d_sw_clrsky_flux_dn_dir, m_buffer.sw_clrsky_flux_dn_dir_k, m_nlay+1); + auto sw_clnsky_flux_up_k = conv.subview2d(d_sw_clnsky_flux_up, m_buffer.sw_clnsky_flux_up_k, m_nlay+1); + auto sw_clnsky_flux_dn_k = conv.subview2d(d_sw_clnsky_flux_dn, m_buffer.sw_clnsky_flux_dn_k, m_nlay+1); + auto sw_clnsky_flux_dn_dir_k = conv.subview2d(d_sw_clnsky_flux_dn_dir, m_buffer.sw_clnsky_flux_dn_dir_k, m_nlay+1); + auto lw_clnclrsky_flux_up_k = conv.subview2d(d_lw_clnclrsky_flux_up, m_buffer.lw_clnclrsky_flux_up_k, m_nlay+1); + auto lw_clnclrsky_flux_dn_k = conv.subview2d(d_lw_clnclrsky_flux_dn, m_buffer.lw_clnclrsky_flux_dn_k, m_nlay+1); + auto lw_clrsky_flux_up_k = conv.subview2d(d_lw_clrsky_flux_up, m_buffer.lw_clrsky_flux_up_k, m_nlay+1); + auto lw_clrsky_flux_dn_k = conv.subview2d(d_lw_clrsky_flux_dn, m_buffer.lw_clrsky_flux_dn_k, m_nlay+1); + auto lw_clnsky_flux_up_k = conv.subview2d(d_lw_clnsky_flux_up, m_buffer.lw_clnsky_flux_up_k, m_nlay+1); + auto lw_clnsky_flux_dn_k = conv.subview2d(d_lw_clnsky_flux_dn, m_buffer.lw_clnsky_flux_dn_k, m_nlay+1); + auto sw_bnd_flux_up_k = m_buffer.sw_bnd_flux_up_k; + auto sw_bnd_flux_dn_k = m_buffer.sw_bnd_flux_dn_k; + auto sw_bnd_flux_dir_k = m_buffer.sw_bnd_flux_dir_k; + auto sw_bnd_flux_dif_k = m_buffer.sw_bnd_flux_dif_k; + auto lw_bnd_flux_up_k = m_buffer.lw_bnd_flux_up_k; + auto lw_bnd_flux_dn_k = m_buffer.lw_bnd_flux_dn_k; + auto sfc_flux_dir_vis_k = conv.subview1d(d_sfc_flux_dir_vis); + auto sfc_flux_dir_nir_k = conv.subview1d(d_sfc_flux_dir_nir); + auto sfc_flux_dif_vis_k = conv.subview1d(d_sfc_flux_dif_vis); + auto sfc_flux_dif_nir_k = conv.subview1d(d_sfc_flux_dif_nir); + auto aero_tau_sw_k = m_buffer.aero_tau_sw_k; + auto aero_ssa_sw_k = m_buffer.aero_ssa_sw_k; + auto aero_g_sw_k = m_buffer.aero_g_sw_k; + auto aero_tau_lw_k = m_buffer.aero_tau_lw_k; + auto cld_tau_sw_bnd_k = conv.subview3d(m_buffer.cld_tau_sw_bnd_k); + auto cld_tau_lw_bnd_k = conv.subview3d(m_buffer.cld_tau_lw_bnd_k); + auto cld_tau_sw_gpt_k = conv.subview3d(m_buffer.cld_tau_sw_gpt_k); + auto cld_tau_lw_gpt_k = conv.subview3d(m_buffer.cld_tau_lw_gpt_k); #endif - }; - - auto p_lay_k = subview_2dk(m_buffer.p_lay_k); - auto t_lay_k = subview_2dk(m_buffer.t_lay_k); - auto p_lev_k = subview_2dk(m_buffer.p_lev_k); - auto z_del_k = subview_2dk(m_buffer.z_del_k); - auto p_del_k = subview_2dk(m_buffer.p_del_k); - auto t_lev_k = subview_2dk(m_buffer.t_lev_k); - auto mu0_k = subview_1dk(m_buffer.mu0_k); - auto sfc_alb_dir_k = subview_2dk(m_buffer.sfc_alb_dir_k); - auto sfc_alb_dif_k = subview_2dk(m_buffer.sfc_alb_dif_k); - auto sfc_alb_dir_vis_k = subview_1dk(m_buffer.sfc_alb_dir_vis_k); - auto sfc_alb_dir_nir_k = subview_1dk(m_buffer.sfc_alb_dir_nir_k); - auto sfc_alb_dif_vis_k = subview_1dk(m_buffer.sfc_alb_dif_vis_k); - auto sfc_alb_dif_nir_k = subview_1dk(m_buffer.sfc_alb_dif_nir_k); - auto qc_k = subview_2dk(m_buffer.qc_k); - auto nc_k = subview_2dk(m_buffer.nc_k); - auto qi_k = subview_2dk(m_buffer.qi_k); - auto cldfrac_tot_k = subview_2dk(m_buffer.cldfrac_tot_k); - auto rel_k = subview_2dk(m_buffer.eff_radius_qc_k); - auto rei_k = subview_2dk(m_buffer.eff_radius_qi_k); - auto sw_flux_up_k = subview_2dk(m_buffer.sw_flux_up_k); - auto sw_flux_dn_k = subview_2dk(m_buffer.sw_flux_dn_k); - auto sw_flux_dn_dir_k = subview_2dk(m_buffer.sw_flux_dn_dir_k); - auto lw_flux_up_k = subview_2dk(m_buffer.lw_flux_up_k); - auto lw_flux_dn_k = subview_2dk(m_buffer.lw_flux_dn_k); - auto sw_clnclrsky_flux_up_k = subview_2dk(m_buffer.sw_clnclrsky_flux_up_k); - auto sw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_k); - auto sw_clnclrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnclrsky_flux_dn_dir_k); - auto sw_clrsky_flux_up_k = subview_2dk(m_buffer.sw_clrsky_flux_up_k); - auto sw_clrsky_flux_dn_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_k); - auto sw_clrsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clrsky_flux_dn_dir_k); - auto sw_clnsky_flux_up_k = subview_2dk(m_buffer.sw_clnsky_flux_up_k); - auto sw_clnsky_flux_dn_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_k); - auto sw_clnsky_flux_dn_dir_k = subview_2dk(m_buffer.sw_clnsky_flux_dn_dir_k); - auto lw_clnclrsky_flux_up_k = subview_2dk(m_buffer.lw_clnclrsky_flux_up_k); - auto lw_clnclrsky_flux_dn_k = subview_2dk(m_buffer.lw_clnclrsky_flux_dn_k); - auto lw_clrsky_flux_up_k = subview_2dk(m_buffer.lw_clrsky_flux_up_k); - auto lw_clrsky_flux_dn_k = subview_2dk(m_buffer.lw_clrsky_flux_dn_k); - auto lw_clnsky_flux_up_k = subview_2dk(m_buffer.lw_clnsky_flux_up_k); - auto lw_clnsky_flux_dn_k = subview_2dk(m_buffer.lw_clnsky_flux_dn_k); - auto sw_bnd_flux_up_k = subview_3dk(m_buffer.sw_bnd_flux_up_k); - auto sw_bnd_flux_dn_k = subview_3dk(m_buffer.sw_bnd_flux_dn_k); - auto sw_bnd_flux_dir_k = subview_3dk(m_buffer.sw_bnd_flux_dir_k); - auto sw_bnd_flux_dif_k = subview_3dk(m_buffer.sw_bnd_flux_dif_k); - auto lw_bnd_flux_up_k = subview_3dk(m_buffer.lw_bnd_flux_up_k); - auto lw_bnd_flux_dn_k = subview_3dk(m_buffer.lw_bnd_flux_dn_k); - auto sfc_flux_dir_vis_k = subview_1dk(m_buffer.sfc_flux_dir_vis_k); - auto sfc_flux_dir_nir_k = subview_1dk(m_buffer.sfc_flux_dir_nir_k); - auto sfc_flux_dif_vis_k = subview_1dk(m_buffer.sfc_flux_dif_vis_k); - auto sfc_flux_dif_nir_k = subview_1dk(m_buffer.sfc_flux_dif_nir_k); - auto aero_tau_sw_k = subview_3dk(m_buffer.aero_tau_sw_k); - auto aero_ssa_sw_k = subview_3dk(m_buffer.aero_ssa_sw_k); - auto aero_g_sw_k = subview_3dk(m_buffer.aero_g_sw_k); - auto aero_tau_lw_k = subview_3dk(m_buffer.aero_tau_lw_k); - auto cld_tau_sw_bnd_k = subview_3dk(m_buffer.cld_tau_sw_bnd_k); - auto cld_tau_lw_bnd_k = subview_3dk(m_buffer.cld_tau_lw_bnd_k); - auto cld_tau_sw_gpt_k = subview_3dk(m_buffer.cld_tau_sw_gpt_k); - auto cld_tau_lw_gpt_k = subview_3dk(m_buffer.cld_tau_lw_gpt_k); -#endif - auto d_tint = m_buffer.d_tint; - auto d_dz = m_buffer.d_dz; - // Set gas concs to "view" only the first ncol columns #ifdef RRTMGP_ENABLE_YAKL @@ -1024,7 +1038,7 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.ncol = ncol; - m_gas_concs_k.concs = subview_3dk(gas_concs_k); + m_gas_concs_k.concs = conv.subview3d(gas_concs_k); #endif // Copy data from the FieldManager to the YAKL arrays @@ -1032,7 +1046,6 @@ void RRTMGPRadiation::run_impl (const double dt) { // Determine the cosine zenith angle // NOTE: Since we are bridging to F90 arrays this must be done on HOST and then // deep copied to a device view. - auto d_mu0 = m_buffer.cosine_zenith; auto h_mu0 = Kokkos::create_mirror_view(d_mu0); if (m_fixed_solar_zenith_angle > 0) { for (int i=0; i(); +#ifdef RRTMGP_ENABLE_KOKKOS + auto tmp2d_k = conv.subview2d_impl(d_vmr, m_nlay); +#endif +#ifdef RRTMGP_ENABLE_YAKL // Copy to YAKL const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol, m_nlay); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { -#ifdef RRTMGP_ENABLE_YAKL tmp2d(i+1,k+1) = d_vmr(icol,k); // Note that for YAKL arrays i and k start with index 1 -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - tmp2d_k(i,k) = d_vmr(icol,k); -#endif }); }); Kokkos::fence(); +#endif // Populate GasConcs object #ifdef RRTMGP_ENABLE_YAKL @@ -1308,12 +1320,12 @@ void RRTMGPRadiation::run_impl (const double dt) { // Compute layer cloud mass (per unit area) #ifdef RRTMGP_ENABLE_YAKL - scream::rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); - scream::rrtmgp::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); + rrtmgp::mixing_ratio_to_cloud_mass(qc, cldfrac_tot, p_del, lwp); + rrtmgp::mixing_ratio_to_cloud_mass(qi, cldfrac_tot, p_del, iwp); #endif #ifdef RRTMGP_ENABLE_KOKKOS - scream::rrtmgp::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); - scream::rrtmgp::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); + interface_t::mixing_ratio_to_cloud_mass(qc_k, cldfrac_tot_k, p_del_k, lwp_k); + interface_t::mixing_ratio_to_cloud_mass(qi_k, cldfrac_tot_k, p_del_k, iwp_k); COMPARE_ALL_WRAP(std::vector({lwp, iwp}), std::vector({lwp_k, iwp_k})); #endif @@ -1329,11 +1341,8 @@ void RRTMGPRadiation::run_impl (const double dt) { iwp(i+1,k+1) *= 1e3; #endif #ifdef RRTMGP_ENABLE_KOKKOS -#ifndef RRTMGP_ENABLE_YAKL - // lwp and lwp_k point to the same memory lwp_k(i,k) *= 1e3; iwp_k(i,k) *= 1e3; -#endif #endif }); }); @@ -1350,7 +1359,7 @@ void RRTMGPRadiation::run_impl (const double dt) { sfc_alb_dir, sfc_alb_dif); #endif #ifdef RRTMGP_ENABLE_KOKKOS - rrtmgp::compute_band_by_band_surface_albedos( + interface_t::compute_band_by_band_surface_albedos( ncol, nswbands, sfc_alb_dir_vis_k, sfc_alb_dir_nir_k, sfc_alb_dif_vis_k, sfc_alb_dif_nir_k, @@ -1384,11 +1393,11 @@ void RRTMGPRadiation::run_impl (const double dt) { ); #endif #ifdef RRTMGP_ENABLE_KOKKOS - rrtmgp::rrtmgp_main( + interface_t::rrtmgp_main( ncol, m_nlay, p_lay_k, t_lay_k, p_lev_k, t_lev_k, m_gas_concs_k, - sfc_alb_dir_k, sfc_alb_dif_k, mu0_k, + sfc_alb_dir_k, sfc_alb_dif_k, d_mu0, lwp_k, iwp_k, rel_k, rei_k, cldfrac_tot_k, aero_tau_sw_k, aero_ssa_sw_k, aero_g_sw_k, aero_tau_lw_k, cld_tau_sw_bnd_k, cld_tau_lw_bnd_k, @@ -1420,7 +1429,6 @@ void RRTMGPRadiation::run_impl (const double dt) { lw_clnclrsky_flux_up_k, lw_clnclrsky_flux_dn_k, lw_clrsky_flux_up_k, lw_clrsky_flux_dn_k, lw_clnsky_flux_up_k, lw_clnsky_flux_dn_k})); - COMPARE_ALL_WRAP(std::vector({sw_bnd_flux_up, sw_bnd_flux_dn, sw_bnd_flux_dir, lw_bnd_flux_up, lw_bnd_flux_dn}), std::vector({sw_bnd_flux_up_k, sw_bnd_flux_dn_k, sw_bnd_flux_dir_k, lw_bnd_flux_up_k, lw_bnd_flux_dn_k})); #endif @@ -1509,7 +1517,7 @@ void RRTMGPRadiation::run_impl (const double dt) { sw_bnd_flux_dif_k(icol,ilev,ibnd) = sw_bnd_flux_dn_k(icol,ilev,ibnd) - sw_bnd_flux_dir_k(icol,ilev,ibnd); }); // Compute surface fluxes - rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot_k, nswbands, sw_bnd_flux_dir_k, sw_bnd_flux_dif_k, sfc_flux_dir_vis_k, sfc_flux_dir_nir_k, @@ -1547,10 +1555,10 @@ void RRTMGPRadiation::run_impl (const double dt) { // does not matter in practice, as clouds probably should not be produced above 50 hPa and we // should not be encountering surface pressure above 1200 hPa, but in the event that things go off // the rails we might want to look at these still. - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 700e2, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldlow_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); - rrtmgp::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 700e2, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldlow_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 400e2, 700e2, p_lay_k, cld_tau_lw_gpt_k, cldmed_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, 400e2, p_lay_k, cld_tau_lw_gpt_k, cldhgh_k); + interface_t::compute_cloud_area(ncol, nlay, nlwgpts, 0, std::numeric_limits::max(), p_lay_k, cld_tau_lw_gpt_k, cldtot_k); COMPARE_ALL_WRAP(std::vector({cldlow, cldmed, cldhgh, cldtot}), std::vector({cldlow_k, cldmed_k, cldhgh_k, cldtot_k})); #endif @@ -1580,9 +1588,9 @@ void RRTMGPRadiation::run_impl (const double dt) { #endif #ifdef RRTMGP_ENABLE_KOKKOS // Get visible 0.67 micron band for COSP - auto idx_067_k = rrtmgp::get_wavelength_index_sw_k(0.67e-6); + auto idx_067_k = interface_t::get_wavelength_index_sw_k(0.67e-6); // Get IR 10.5 micron band for COSP - auto idx_105_k = rrtmgp::get_wavelength_index_lw_k(10.5e-6); + auto idx_105_k = interface_t::get_wavelength_index_lw_k(10.5e-6); real1dk T_mid_at_cldtop_k (d_T_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); real1dk p_mid_at_cldtop_k (d_p_mid_at_cldtop.data() + m_col_chunk_beg[ic], ncol); @@ -1593,7 +1601,7 @@ void RRTMGPRadiation::run_impl (const double dt) { real1dk eff_radius_qc_at_cldtop_k (d_eff_radius_qc_at_cldtop.data() + m_col_chunk_beg[ic], ncol); real1dk eff_radius_qi_at_cldtop_k (d_eff_radius_qi_at_cldtop.data() + m_col_chunk_beg[ic], ncol); - rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, t_lay_k, p_lay_k, p_del_k, z_del_k, qc_k, qi_k, rel_k, rei_k, cldfrac_tot_k, nc_k, T_mid_at_cldtop_k, p_mid_at_cldtop_k, cldfrac_ice_at_cldtop_k, cldfrac_liq_at_cldtop_k, cldfrac_tot_at_cldtop_k, cdnc_at_cldtop_k, @@ -1658,12 +1666,10 @@ void RRTMGPRadiation::run_impl (const double dt) { Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { const int i = team.league_rank(); const int icol = i + beg; - d_sfc_flux_dir_nir(icol) = sfc_flux_dir_nir_k(i); - d_sfc_flux_dir_vis(icol) = sfc_flux_dir_vis_k(i); - d_sfc_flux_dif_nir(icol) = sfc_flux_dif_nir_k(i); - d_sfc_flux_dif_vis(icol) = sfc_flux_dif_vis_k(i); d_sfc_flux_sw_net(icol) = sw_flux_dn_k(i,kbot_k) - sw_flux_up_k(i,kbot_k); d_sfc_flux_lw_dn(icol) = lw_flux_dn_k(i,kbot_k); +#ifdef RRTMGP_LAYOUT_LEFT + // Copy from layout left buffer views back to layout right fields Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { d_sw_flux_up(icol,k) = sw_flux_up_k(i,k); d_sw_flux_dn(icol,k) = sw_flux_dn_k(i,k); @@ -1686,6 +1692,7 @@ void RRTMGPRadiation::run_impl (const double dt) { d_lw_clnsky_flux_up(icol,k) = lw_clnsky_flux_up_k(i,k); d_lw_clnsky_flux_dn(icol,k) = lw_clnsky_flux_dn_k(i,k); }); +#endif // Extract optical properties for COSP Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { d_dtau067(icol,k) = cld_tau_sw_bnd_k(i,k,idx_067_k); @@ -1769,11 +1776,12 @@ void RRTMGPRadiation::run_impl (const double dt) { void RRTMGPRadiation::finalize_impl () { #ifdef RRTMGP_ENABLE_YAKL m_gas_concs.reset(); + rrtmgp::rrtmgp_finalize(); #endif #ifdef RRTMGP_ENABLE_KOKKOS m_gas_concs_k.reset(); + interface_t::rrtmgp_finalize(); #endif - rrtmgp::rrtmgp_finalize(); finalize_kls(); } diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 66c89476028b..329c7eeaba8d 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -14,19 +14,41 @@ namespace scream { * exactly ONE instance of this class in its list of subcomponents. */ +// rrtmgp is performance tuned for layout left views but will accept any +// view. We probably want to stick with layout left views for performance +// reasons even though this requires us to make copies of our fields (they +// are layout right). +#define RRTMGP_LAYOUT_LEFT + class RRTMGPRadiation : public AtmosphereProcess { public: - using view_1d_real = typename ekat::KokkosTypes::template view_1d; - using view_2d_real = typename ekat::KokkosTypes::template view_2d; - using view_3d_real = typename ekat::KokkosTypes::template view_3d; - using view_2d_real_const = typename ekat::KokkosTypes::template view_2d; - using ci_string = ekat::CaseInsensitiveString; - - using KT = ekat::KokkosTypes; - template - using uview_1d = Unmanaged>; - template - using uview_2d = Unmanaged>; + using KT = ekat::KokkosTypes; +#ifdef RRTMGP_LAYOUT_LEFT + using layout_t = Kokkos::LayoutLeft; +#else + using layout_t = typename ekat::KokkosTypes::Layout; +#endif + using real1dk = Kokkos::View; + using real2dk = Kokkos::View; + using real3dk = Kokkos::View; + using creal1dk = Kokkos::View; + using creal2dk = Kokkos::View; + using creal3dk = Kokkos::View; + using ureal1dk = Unmanaged; + using ureal2dk = Unmanaged; + using ureal3dk = Unmanaged; + using cureal1dk = Unmanaged; + using cureal2dk = Unmanaged; + using cureal3dk = Unmanaged; + + using ci_string = ekat::CaseInsensitiveString; + + using lrreal2dk = typename KT::template view_2d; + using ulrreal2dk = Unmanaged; + +#ifdef RRTMGP_ENABLE_KOKKOS + using interface_t = rrtmgp::rrtmgp_interface; +#endif // Constructors RRTMGPRadiation (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -87,12 +109,12 @@ class RRTMGPRadiation : public AtmosphereProcess { // These are the gases that we keep track of int m_ngas; std::vector m_gas_names; - view_1d_real m_gas_mol_weights; + real1dk m_gas_mol_weights; #ifdef RRTMGP_ENABLE_YAKL GasConcs m_gas_concs; #endif #ifdef RRTMGP_ENABLE_KOKKOS - GasConcsK m_gas_concs_k; + GasConcsK m_gas_concs_k; #endif // Prescribed greenhouse gas surface concentrations in moles / moles air @@ -124,7 +146,7 @@ class RRTMGPRadiation : public AtmosphereProcess { static constexpr int num_3d_nlay_nlwgpts = 1; // 1d size (ncol) - uview_1d cosine_zenith; + ureal1dk cosine_zenith; #ifdef RRTMGP_ENABLE_YAKL real1d mu0; real1d sfc_alb_dir_vis; @@ -137,19 +159,19 @@ class RRTMGPRadiation : public AtmosphereProcess { real1d sfc_flux_dif_nir; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real1dk mu0_k; - real1dk sfc_alb_dir_vis_k; - real1dk sfc_alb_dir_nir_k; - real1dk sfc_alb_dif_vis_k; - real1dk sfc_alb_dif_nir_k; - real1dk sfc_flux_dir_vis_k; - real1dk sfc_flux_dir_nir_k; - real1dk sfc_flux_dif_vis_k; - real1dk sfc_flux_dif_nir_k; + ureal1dk mu0_k; + ureal1dk sfc_alb_dir_vis_k; + ureal1dk sfc_alb_dir_nir_k; + ureal1dk sfc_alb_dif_vis_k; + ureal1dk sfc_alb_dif_nir_k; + ureal1dk sfc_flux_dir_vis_k; + ureal1dk sfc_flux_dir_nir_k; + ureal1dk sfc_flux_dif_vis_k; + ureal1dk sfc_flux_dif_nir_k; #endif // 2d size (ncol, nlay) - uview_2d d_dz; + ureal2dk d_dz; #ifdef RRTMGP_ENABLE_YAKL real2d p_lay; real2d t_lay; @@ -168,25 +190,25 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_heating; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lay_k; - real2dk t_lay_k; - real2dk z_del_k; - real2dk p_del_k; - real2dk qc_k; - real2dk nc_k; - real2dk qi_k; - real2dk cldfrac_tot_k; - real2dk eff_radius_qc_k; - real2dk eff_radius_qi_k; - real2dk tmp2d_k; - real2dk lwp_k; - real2dk iwp_k; - real2dk sw_heating_k; - real2dk lw_heating_k; + ureal2dk p_lay_k; + ureal2dk t_lay_k; + ureal2dk z_del_k; + ureal2dk p_del_k; + ureal2dk qc_k; + ureal2dk nc_k; + ureal2dk qi_k; + ureal2dk cldfrac_tot_k; + ureal2dk eff_radius_qc_k; + ureal2dk eff_radius_qi_k; + ureal2dk tmp2d_k; + ureal2dk lwp_k; + ureal2dk iwp_k; + ureal2dk sw_heating_k; + ureal2dk lw_heating_k; #endif // 2d size (ncol, nlay+1) - uview_2d d_tint; + ureal2dk d_tint; #ifdef RRTMGP_ENABLE_YAKL real2d p_lev; real2d t_lev; @@ -212,28 +234,28 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d lw_clnsky_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk p_lev_k; - real2dk t_lev_k; - real2dk sw_flux_up_k; - real2dk sw_flux_dn_k; - real2dk sw_flux_dn_dir_k; - real2dk lw_flux_up_k; - real2dk lw_flux_dn_k; - real2dk sw_clnclrsky_flux_up_k; - real2dk sw_clnclrsky_flux_dn_k; - real2dk sw_clnclrsky_flux_dn_dir_k; - real2dk sw_clrsky_flux_up_k; - real2dk sw_clrsky_flux_dn_k; - real2dk sw_clrsky_flux_dn_dir_k; - real2dk sw_clnsky_flux_up_k; - real2dk sw_clnsky_flux_dn_k; - real2dk sw_clnsky_flux_dn_dir_k; - real2dk lw_clnclrsky_flux_up_k; - real2dk lw_clnclrsky_flux_dn_k; - real2dk lw_clrsky_flux_up_k; - real2dk lw_clrsky_flux_dn_k; - real2dk lw_clnsky_flux_up_k; - real2dk lw_clnsky_flux_dn_k; + ureal2dk p_lev_k; + ureal2dk t_lev_k; + ureal2dk sw_flux_up_k; + ureal2dk sw_flux_dn_k; + ureal2dk sw_flux_dn_dir_k; + ureal2dk lw_flux_up_k; + ureal2dk lw_flux_dn_k; + ureal2dk sw_clnclrsky_flux_up_k; + ureal2dk sw_clnclrsky_flux_dn_k; + ureal2dk sw_clnclrsky_flux_dn_dir_k; + ureal2dk sw_clrsky_flux_up_k; + ureal2dk sw_clrsky_flux_dn_k; + ureal2dk sw_clrsky_flux_dn_dir_k; + ureal2dk sw_clnsky_flux_up_k; + ureal2dk sw_clnsky_flux_dn_k; + ureal2dk sw_clnsky_flux_dn_dir_k; + ureal2dk lw_clnclrsky_flux_up_k; + ureal2dk lw_clnclrsky_flux_dn_k; + ureal2dk lw_clrsky_flux_up_k; + ureal2dk lw_clrsky_flux_dn_k; + ureal2dk lw_clnsky_flux_up_k; + ureal2dk lw_clnsky_flux_dn_k; #endif // 3d size (ncol, nlay+1, nswbands) @@ -244,10 +266,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d sw_bnd_flux_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk sw_bnd_flux_up_k; - real3dk sw_bnd_flux_dn_k; - real3dk sw_bnd_flux_dir_k; - real3dk sw_bnd_flux_dif_k; + ureal3dk sw_bnd_flux_up_k; + ureal3dk sw_bnd_flux_dn_k; + ureal3dk sw_bnd_flux_dir_k; + ureal3dk sw_bnd_flux_dif_k; #endif // 3d size (ncol, nlay+1, nlwbands) @@ -256,8 +278,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d lw_bnd_flux_dn; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk lw_bnd_flux_up_k; - real3dk lw_bnd_flux_dn_k; + ureal3dk lw_bnd_flux_up_k; + ureal3dk lw_bnd_flux_dn_k; #endif // 2d size (ncol, nswbands) @@ -266,8 +288,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real2d sfc_alb_dif; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real2dk sfc_alb_dir_k; - real2dk sfc_alb_dif_k; + ureal2dk sfc_alb_dir_k; + ureal2dk sfc_alb_dif_k; #endif // 3d size (ncol, nlay, n[sw,lw]bands) @@ -278,10 +300,10 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d aero_tau_lw; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk aero_tau_sw_k; - real3dk aero_ssa_sw_k; - real3dk aero_g_sw_k; - real3dk aero_tau_lw_k; + ureal3dk aero_tau_sw_k; + ureal3dk aero_ssa_sw_k; + ureal3dk aero_g_sw_k; + ureal3dk aero_tau_lw_k; #endif // 3d size (ncol, nlay, n[sw,lw]bnds) @@ -290,8 +312,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_bnd; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_bnd_k; - real3dk cld_tau_lw_bnd_k; + ureal3dk cld_tau_sw_bnd_k; + ureal3dk cld_tau_lw_bnd_k; #endif // 3d size (ncol, nlay, n[sw,lw]gpts) @@ -300,8 +322,8 @@ class RRTMGPRadiation : public AtmosphereProcess { real3d cld_tau_lw_gpt; #endif #ifdef RRTMGP_ENABLE_KOKKOS - real3dk cld_tau_sw_gpt_k; - real3dk cld_tau_lw_gpt_k; + ureal3dk cld_tau_sw_gpt_k; + ureal3dk cld_tau_lw_gpt_k; #endif }; diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp index 54a32133da92..65cc99e09cc3 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.cpp @@ -1,25 +1,13 @@ -#include "physics/rrtmgp/scream_rrtmgp_interface.hpp" #include "physics/rrtmgp/rrtmgp_test_utils.hpp" #ifdef RRTMGP_ENABLE_YAKL #include "YAKL_netcdf.h" #endif -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "cpp/rte/mo_fluxes.h" -#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" - #include #include namespace rrtmgpTest { -#ifdef RRTMGP_ENABLE_YAKL -using yakl::fortran::parallel_for; -using yakl::fortran::SimpleBounds; -using yakl::intrinsics::mod; -using yakl::intrinsics::merge; -#endif - bool file_exists(const char *filename) { if (auto file = fopen(filename, "r")) { fclose(file); @@ -29,8 +17,12 @@ bool file_exists(const char *filename) { } } -// TODO: use YAKL intrinsics for this to avoid needing to make host copies #ifdef RRTMGP_ENABLE_YAKL +using yakl::fortran::parallel_for; +using yakl::fortran::SimpleBounds; +using yakl::intrinsics::mod; +using yakl::intrinsics::merge; + bool all_close(real2d &arr1, real2d &arr2, double tolerance) { int nx = arr1.dimension[0]; int ny = arr2.dimension[1]; @@ -46,28 +38,7 @@ bool all_close(real2d &arr1, real2d &arr2, double tolerance) { } return true; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) { - int nx = arr1.extent(0); - int ny = arr2.extent(1); - auto arr1_h = Kokkos::create_mirror_view(arr1); - auto arr2_h = Kokkos::create_mirror_view(arr2); - Kokkos::deep_copy(arr1_h, arr1); - Kokkos::deep_copy(arr2_h, arr2); - for (int i=0; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { - printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); - return false; - } - } - } - return true; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void dummy_atmos( std::string inputfile, int ncol, real2d &p_lay, real2d &t_lay, @@ -94,37 +65,7 @@ void dummy_atmos( // create a dummy atmosphere. dummy_clouds(scream::rrtmgp::cloud_optics_sw, p_lay, t_lay, lwp, iwp, rel, rei, cld); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void dummy_atmos( - std::string inputfile, - int ncol, real2dk &p_lay, real2dk &t_lay, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld) { - - // Setup boundary conditions, solar zenith angle, etc - // NOTE: this stuff would come from the model in a real run - // Ocean-ish values for surface albedos, just for example - Kokkos::deep_copy(sfc_alb_dir_vis , 0.06 ); - Kokkos::deep_copy(sfc_alb_dir_nir , 0.06 ); - Kokkos::deep_copy(sfc_alb_dif_vis , 0.06 ); - Kokkos::deep_copy(sfc_alb_dif_nir , 0.06 ); - - // Pick a solar zenith angle; this should come from the model - Kokkos::deep_copy(mu0, 0.86 ); - - // Get dummy cloud PHYSICAL properties. Note that this function call - // needs the CloudOptics object only because it uses the min and max - // valid values from the lookup tables for liquid and ice water path to - // create a dummy atmosphere. - dummy_clouds(scream::rrtmgp::cloud_optics_sw_k, p_lay, t_lay, lwp, iwp, rel, rei, cld); -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL void dummy_clouds( CloudOptics &cloud_optics, real2d &p_lay, real2d &t_lay, real2d &lwp, real2d &iwp, real2d &rel, real2d &rei, real2d &cloud_mask) { @@ -151,38 +92,7 @@ void dummy_clouds( rei(icol,ilay) = merge(rei_val, 0._wp, iwp(icol,ilay) > 0._wp); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void dummy_clouds( - CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cloud_mask) { - - // Problem sizes - int ncol = t_lay.extent(0); - int nlay = t_lay.extent(1); - - // Generate some fake liquid and ice water data. We pick values to be midway between - // the min and max of the valid lookup table values for effective radii - real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); - real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); - // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and - // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. - // Set sane values for liquid and ice water path. - // NOTE: these "sane" values are in g/m2! - Kokkos::parallel_for( conv::get_mdrp<2>({nlay,ncol}) , KOKKOS_LAMBDA (int ilay, int icol) { - cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100. * 100. && p_lay(icol,ilay) < 900. * 100. && ((icol+1)%3) != 0; - // Ice and liquid will overlap in a few layers - lwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263.); - iwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273.); - rel(icol,ilay) = conv::merge(rel_val, 0., lwp(icol,ilay) > 0.); - rei(icol,ilay) = conv::merge(rei_val, 0., iwp(icol,ilay) > 0.); - }); -} -#endif - -// Function to read fluxes from input file so we can compare our answers against the reference -#ifdef RRTMGP_ENABLE_YAKL void read_fluxes( std::string inputfile, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, @@ -208,36 +118,7 @@ void read_fluxes( io.read(lw_flux_up, "lw_flux_up" ); io.read(lw_flux_dn, "lw_flux_dn" ); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void read_fluxes( - std::string inputfile, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn) { - - // Initialize netcdf reader - conv::SimpleNetCDF io; - io.open(inputfile, NC_NOWRITE); - - // Initialize arrays to hold fluxes - int nlev = io.getDimSize("lev"); - int ncol = io.getDimSize("col_flx"); - sw_flux_up = real2dk("sw_flux_up" , ncol, nlev); - sw_flux_dn = real2dk("sw_flux_dn" , ncol, nlev); - sw_flux_dir = real2dk("sw_flux_dir", ncol, nlev); - lw_flux_up = real2dk("lw_flux_up" , ncol, nlev); - lw_flux_dn = real2dk("lw_flux_dn" , ncol, nlev); - - // Read data - io.read(sw_flux_up, "sw_flux_up" ); - io.read(sw_flux_dn, "sw_flux_dn" ); - io.read(sw_flux_dir, "sw_flux_dir"); - io.read(lw_flux_up, "lw_flux_up" ); - io.read(lw_flux_dn, "lw_flux_dn" ); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void write_fluxes( std::string outputfile, real2d &sw_flux_up, real2d &sw_flux_dn, real2d &sw_flux_dir, @@ -253,76 +134,5 @@ void write_fluxes( io.close(); } #endif -#ifdef RRTMGP_ENABLE_KOKKOS -void write_fluxes( - std::string outputfile, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn) { - - conv::SimpleNetCDF io; - io.create(outputfile); - io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); - io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); - io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); - io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); - io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); - io.close(); -} -#endif - -// TODO: This function should instead take values, not file names, -// because for the test we do not want to write to file -#ifdef RRTMGP_ENABLE_YAKL -int compare_y(std::string file1, std::string file2) { - // Read data from baseline and test file - real2d sw_flux_up_1; - real2d sw_flux_dn_1; - real2d sw_flux_dir_1; - real2d lw_flux_up_1; - real2d lw_flux_dn_1; - read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); - real2d sw_flux_up_2; - real2d sw_flux_dn_2; - real2d sw_flux_dir_2; - real2d lw_flux_up_2; - real2d lw_flux_dn_2; - read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); - - // Check values - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; - return nerr; -} -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int compare_k(std::string file1, std::string file2) { - // Read data from baseline and test file - real2dk sw_flux_up_1; - real2dk sw_flux_dn_1; - real2dk sw_flux_dir_1; - real2dk lw_flux_up_1; - real2dk lw_flux_dn_1; - read_fluxes(file1, sw_flux_up_1, sw_flux_dn_1, sw_flux_dir_1, lw_flux_up_1, lw_flux_dn_1); - real2dk sw_flux_up_2; - real2dk sw_flux_dn_2; - real2dk sw_flux_dir_2; - real2dk lw_flux_up_2; - real2dk lw_flux_dn_2; - read_fluxes(file2, sw_flux_up_2, sw_flux_dn_2, sw_flux_dir_2, lw_flux_up_2, lw_flux_dn_2); - - // Check values - int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_1 , sw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_1 , sw_flux_dn_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_1 , sw_flux_dir_2, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_1 , lw_flux_up_2 , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_1 , lw_flux_dn_2 , 0.001)) nerr++; - return nerr; -} -#endif } // namespace rrtmgp diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp index b9498a9a34f9..466467919fcf 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_test_utils.hpp @@ -1,5 +1,10 @@ #ifndef RRTMGP_TEST_UTILS_HPP #define RRTMGP_TEST_UTILS_HPP + +#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" +#include "physics/rrtmgp/scream_rrtmgp_interface.hpp" +#include "cpp/rrtmgp/mo_gas_concentrations.h" +#include "cpp/rte/mo_fluxes.h" #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" namespace rrtmgpTest { @@ -37,33 +42,134 @@ void write_fluxes( #endif #ifdef RRTMGP_ENABLE_KOKKOS -bool all_close(real2dk &arr1, real2dk &arr2, double tolerance); +template +struct rrtmgp_test_utils { -void dummy_clouds( - CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld -); +using interface_t = scream::rrtmgp::rrtmgp_interface; +using real1dk = typename interface_t::view_t; +using real2dk = typename interface_t::view_t; +using real3dk = typename interface_t::view_t; +using MDRP = typename conv::MDRP; -void dummy_atmos( +static bool all_close(real2dk &arr1, real2dk &arr2, double tolerance) +{ + int nx = arr1.extent(0); + int ny = arr2.extent(1); + auto arr1_h = Kokkos::create_mirror_view(arr1); + auto arr2_h = Kokkos::create_mirror_view(arr2); + Kokkos::deep_copy(arr1_h, arr1); + Kokkos::deep_copy(arr2_h, arr2); + for (int i=0; i tolerance || std::isnan(arr1_h(i,j) - arr2_h(i,j))) { + printf("arr1 = %f, arr2 = %f at %i,%i\n", arr1_h(i,j), arr2_h(i,j), i, j); + return false; + } + } + } + return true; +} + +static void dummy_clouds( + CloudOpticsK &cloud_optics, real2dk &p_lay, real2dk &t_lay, + real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cloud_mask +) +{ + // Problem sizes + int ncol = t_lay.extent(0); + int nlay = t_lay.extent(1); + + // Generate some fake liquid and ice water data. We pick values to be midway between + // the min and max of the valid lookup table values for effective radii + real rel_val = 0.5 * (cloud_optics.get_min_radius_liq() + cloud_optics.get_max_radius_liq()); + real rei_val = 0.5 * (cloud_optics.get_min_radius_ice() + cloud_optics.get_max_radius_ice()); + + // Restrict clouds to troposphere (> 100 hPa = 100*100 Pa) and not very close to the ground (< 900 hPa), and + // put them in 2/3 of the columns since that's roughly the total cloudiness of earth. + // Set sane values for liquid and ice water path. + // NOTE: these "sane" values are in g/m2! + Kokkos::parallel_for( MDRP::template get<2>({nlay,ncol}) , KOKKOS_LAMBDA (int ilay, int icol) { + cloud_mask(icol,ilay) = p_lay(icol,ilay) > 100. * 100. && p_lay(icol,ilay) < 900. * 100. && ((icol+1)%3) != 0; + // Ice and liquid will overlap in a few layers + lwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) > 263.); + iwp(icol,ilay) = conv::merge(10., 0., cloud_mask(icol,ilay) && t_lay(icol,ilay) < 273.); + rel(icol,ilay) = conv::merge(rel_val, 0., lwp(icol,ilay) > 0.); + rei(icol,ilay) = conv::merge(rei_val, 0., iwp(icol,ilay) > 0.); + }); +} + +static void dummy_atmos( std::string inputfile, int ncol, real2dk &p_lay, real2dk &t_lay, real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, real1dk &mu0, real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cld -); +) +{ + // Setup boundary conditions, solar zenith angle, etc + // NOTE: this stuff would come from the model in a real run -void read_fluxes( + // Ocean-ish values for surface albedos, just for example + Kokkos::deep_copy(sfc_alb_dir_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dir_nir , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_vis , 0.06 ); + Kokkos::deep_copy(sfc_alb_dif_nir , 0.06 ); + + // Pick a solar zenith angle; this should come from the model + Kokkos::deep_copy(mu0, 0.86 ); + + // Get dummy cloud PHYSICAL properties. Note that this function call + // needs the CloudOptics object only because it uses the min and max + // valid values from the lookup tables for liquid and ice water path to + // create a dummy atmosphere. + dummy_clouds(interface_t::cloud_optics_sw_k, p_lay, t_lay, lwp, iwp, rel, rei, cld); +} + +static void read_fluxes( std::string inputfile, real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, real2dk &lw_flux_up, real2dk &lw_flux_dn -); +) +{ + // Initialize netcdf reader + conv::SimpleNetCDF io; + io.open(inputfile, NC_NOWRITE); -void write_fluxes( + // Initialize arrays to hold fluxes + int nlev = io.getDimSize("lev"); + int ncol = io.getDimSize("col_flx"); + sw_flux_up = real2dk("sw_flux_up" , ncol, nlev); + sw_flux_dn = real2dk("sw_flux_dn" , ncol, nlev); + sw_flux_dir = real2dk("sw_flux_dir", ncol, nlev); + lw_flux_up = real2dk("lw_flux_up" , ncol, nlev); + lw_flux_dn = real2dk("lw_flux_dn" , ncol, nlev); + + // Read data + io.read(sw_flux_up, "sw_flux_up" ); + io.read(sw_flux_dn, "sw_flux_dn" ); + io.read(sw_flux_dir, "sw_flux_dir"); + io.read(lw_flux_up, "lw_flux_up" ); + io.read(lw_flux_dn, "lw_flux_dn" ); +} + +static void write_fluxes( std::string outputfile, real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dir, real2dk &lw_flux_up, real2dk &lw_flux_dn -); +) +{ + conv::SimpleNetCDF io; + io.create(outputfile); + io.write(sw_flux_up , "sw_flux_up" , {"col_flx","lev"}); + io.write(sw_flux_dn , "sw_flux_dn" , {"col_flx","lev"}); + io.write(sw_flux_dir, "sw_flux_dir", {"col_flx","lev"}); + io.write(lw_flux_up , "lw_flux_up" , {"col_flx","lev"}); + io.write(lw_flux_dn , "lw_flux_dn" , {"col_flx","lev"}); + io.close(); +} + +}; #endif } diff --git a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp index 0dfe4945e0c8..aa6326c81811 100644 --- a/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp +++ b/components/eamxx/src/physics/rrtmgp/rrtmgp_utils.hpp @@ -57,9 +57,10 @@ void compute_heating_rate ( View4 const &heating_rate) { using physconst = scream::physics::Constants; + using MDRP = typename conv::MDRP; auto ncol = flux_up.extent(0); auto nlay = flux_up.extent(1)-1; - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { heating_rate(icol,ilay) = ( flux_up(icol,ilay+1) - flux_up(icol,ilay) - flux_dn(icol,ilay+1) + flux_dn(icol,ilay) @@ -98,28 +99,31 @@ bool check_range(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=s } }); auto num_bad = sum(bad_mask); - pass = false; - out << msg << ": " - << num_bad << " values outside range " - << "[" << xmin << "," << xmax << "]" - << "; minval = " << _xmin - << "; maxval = " << _xmax << "\n"; + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } } return pass; } #endif #ifdef RRTMGP_ENABLE_KOKKOS -template -bool check_range_k(T x, Real xmin, Real xmax, std::string msg, std::ostream& out=std::cout) { +template ::type* dummy = nullptr> +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { bool pass = true; auto _xmin = conv::minval(x); auto _xmax = conv::maxval(x); if (_xmin < xmin or _xmax > xmax) { // How many outside range? - bool1dk bad_mask("bad_mask", x.size()); - Kokkos::parallel_for(x.size(), KOKKOS_LAMBDA (int i) { - if (x.data()[i] < xmin or x.data()[i] > xmax) { - bad_mask.data()[i] = true; + Kokkos::View bad_mask("bad_mask", x.extent(0)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + if (x(i) < xmin or x(i) > xmax) { + bad_mask(i) = true; } }); auto num_bad = conv::sum(bad_mask); @@ -132,6 +136,68 @@ bool check_range_k(T x, Real xmin, Real xmax, std::string msg, std::ostream& out } return pass; } + +template ::type* dummy = nullptr> +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = conv::minval(x); + auto _xmax = conv::maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + Kokkos::View bad_mask("bad_mask", x.extent(0), x.extent(1)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + for (size_t j = 0; j < x.extent(1); ++j) { + if (x(i, j) < xmin or x(i, j) > xmax) { + bad_mask(i, j) = true; + } + } + }); + auto num_bad = conv::sum(bad_mask); + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + } + return pass; +} + +template ::type* dummy = nullptr> +bool check_range_k(T x, typename T::const_value_type xmin, typename T::const_value_type xmax, + std::string msg, std::ostream& out=std::cout) { + bool pass = true; + auto _xmin = conv::minval(x); + auto _xmax = conv::maxval(x); + if (_xmin < xmin or _xmax > xmax) { + // How many outside range? + Kokkos::View bad_mask("bad_mask", x.extent(0), x.extent(1), x.extent(2)); + Kokkos::parallel_for(x.extent(0), KOKKOS_LAMBDA (int i) { + for (size_t j = 0; j < x.extent(1); ++j) { + for (size_t k = 0; k < x.extent(2); ++k) { + if (x(i, j, k) < xmin or x(i, j, k) > xmax) { + bad_mask(i, j, k) = true; + } + } + } + }); + auto num_bad = conv::sum(bad_mask); + if (num_bad > 0) { + pass = false; + out << msg << ": " + << num_bad << " values outside range " + << "[" << xmin << "," << xmax << "]" + << "; minval = " << _xmin + << "; maxval = " << _xmax << "\n"; + } + } + return pass; +} + + #endif } // namespace rrtmgp diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp index 450d7ad216be..10281df8078c 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.cpp @@ -1,17 +1,5 @@ #include "scream_rrtmgp_interface.hpp" -#include "rrtmgp_utils.hpp" -#include "cpp/examples/mo_load_coefficients.h" -#include "examples/all-sky/mo_load_cloud_coefficients.h" -#include "cpp/rrtmgp/mo_gas_concentrations.h" -#include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" -#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" -#include "cpp/rte/mo_rte_sw.h" -#include "cpp/rte/mo_rte_lw.h" #include "physics/share/physics_constants.hpp" -#include "ekat/util/ekat_math_utils.hpp" -#ifdef RRTMGP_ENABLE_KOKKOS -#include "Kokkos_Random.hpp" -#endif namespace scream { @@ -38,41 +26,27 @@ void finalize_kls() #endif } +#ifdef RRTMGP_ENABLE_YAKL namespace rrtmgp { -#ifdef RRTMGP_ENABLE_YAKL using yakl::fortran::parallel_for; using yakl::fortran::SimpleBounds; using yakl::intrinsics::merge; -#endif - /* * Objects containing k-distribution information need to be initialized * once and then persist throughout the life of the program, so we * declare them here within the rrtmgp namespace. */ -#ifdef RRTMGP_ENABLE_YAKL GasOpticsRRTMGP k_dist_sw; GasOpticsRRTMGP k_dist_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -GasOpticsRRTMGPK k_dist_sw_k; -GasOpticsRRTMGPK k_dist_lw_k; -#endif /* * Objects containing cloud optical property look-up table information. * We want to initialize these once and use throughout the life of the * program, so declare here and read data in during rrtmgp_initialize(). */ -#ifdef RRTMGP_ENABLE_YAKL CloudOptics cloud_optics_sw; CloudOptics cloud_optics_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -CloudOpticsK cloud_optics_sw_k; -CloudOpticsK cloud_optics_lw_k; -#endif bool initialized = false; bool initialized_k = false; @@ -80,7 +54,6 @@ bool initialized_k = false; // local functions namespace { -#ifdef RRTMGP_ENABLE_YAKL OpticalProps2str get_cloud_optics_sw( const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, @@ -106,36 +79,7 @@ OpticalProps2str get_cloud_optics_sw( // Return optics return clouds; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps2strK get_cloud_optics_sw( - const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { - - // Initialize optics - OpticalProps2strK clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_2str(ncol, nlay); - - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); - - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2dk("rel_limited", ncol, nlay); - auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); - - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); - - // Return optics - return clouds; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps1scl get_cloud_optics_lw( const int ncol, const int nlay, CloudOptics &cloud_optics, GasOpticsRRTMGP &kdist, @@ -161,36 +105,7 @@ OpticalProps1scl get_cloud_optics_lw( // Return optics return clouds; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps1sclK get_cloud_optics_lw( - const int ncol, const int nlay, - CloudOpticsK &cloud_optics, GasOpticsRRTMGPK &kdist, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei) { - - // Initialize optics - OpticalProps1sclK clouds; - clouds.init(kdist.get_band_lims_wavenumber()); - clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! - - // Needed for consistency with all-sky example problem? - cloud_optics.set_ice_roughness(2); - - // Limit effective radii to be within bounds of lookup table - auto rel_limited = real2dk("rel_limited", ncol, nlay); - auto rei_limited = real2dk("rei_limited", ncol, nlay); - limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); - limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); - - // Calculate cloud optics - cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); - - // Return optics - return clouds; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps2str get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps2str &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { @@ -242,60 +157,7 @@ OpticalProps2str get_subsampled_clouds( }); return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps2strK get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps2strK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { - // Initialized subsampled optics - OpticalProps2strK subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_2str(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, - // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), - // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should - // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very - // high resolution. - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = int1dk("seeds", ncol); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); - subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - subsampled_optics.ssa(icol,ilay,igpt) = 0; - subsampled_optics.g (icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL OpticalProps1scl get_subsampled_clouds( const int ncol, const int nlay, const int nbnd, const int ngpt, OpticalProps1scl &cloud_optics, GasOpticsRRTMGP &kdist, real2d &cld, real2d &p_lay) { @@ -339,51 +201,6 @@ OpticalProps1scl get_subsampled_clouds( }); return subsampled_optics; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -OpticalProps1sclK get_subsampled_clouds( - const int ncol, const int nlay, const int nbnd, const int ngpt, - OpticalProps1sclK &cloud_optics, GasOpticsRRTMGPK &kdist, real2dk &cld, real2dk &p_lay) { - // Initialized subsampled optics - OpticalProps1sclK subsampled_optics; - subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); - subsampled_optics.alloc_1scl(ncol, nlay); - // Check that we do not have clouds with no optical properties; this would get corrected - // when we assign optical props, but we want to use a "radiative cloud fraction" - // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud - // mask profiles with no actual cloud properties in between, which would just further overestimate - // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped - // even when separated by layers with no cloud properties, when in fact those layers should be - // randomly overlapped. - auto cldfrac_rad = real2dk("cldfrac_rad", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - if (cloud_optics.tau(icol,ilay,ibnd) > 0) { - cldfrac_rad(icol,ilay) = cld(icol,ilay); - } - }); - // Get subcolumn cloud mask - int overlap = 1; - // Get unique seeds for each column that are reproducible across different MPI rank layouts; - // use decimal part of pressure for this, consistent with the implementation in EAM; use different - // seed values for longwave and shortwave - auto seeds = int1dk("seeds", ncol); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); - }); - auto cldmask = get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds); - // Assign optical properties to subcolumns (note this implements MCICA) - auto gpoint_bands = kdist.get_gpoint_bands(); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - auto ibnd = gpoint_bands(igpt); - if (cldmask(icol,ilay,igpt) == 1) { - subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); - } else { - subsampled_optics.tau(icol,ilay,igpt) = 0; - } - }); - return subsampled_optics; -} -#endif } @@ -392,7 +209,6 @@ OpticalProps1sclK get_subsampled_clouds( * can be used as-is, but are intended to be wrapped by the SCREAM AD * interface to radiation. */ -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_initialize(GasConcs &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, @@ -419,65 +235,15 @@ void rrtmgp_initialize(GasConcs &gas_concs, // We are now initialized! initialized = true; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_initialize( - GasConcsK &gas_concs, - const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, - const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, - const std::shared_ptr& logger) -{ - // If we've already initialized, just exit - if (initialized_k) { - if (logger) - logger->info("RRTMGP is already initialized; skipping\n"); - return; - } - - // Initialize Kokkos - if (!Kokkos::is_initialized()) { Kokkos::initialize(); } - - // Load and initialize absorption coefficient data - load_and_init(k_dist_sw_k, coefficients_file_sw, gas_concs); - load_and_init(k_dist_lw_k, coefficients_file_lw, gas_concs); - - // Load and initialize cloud optical property look-up table information - load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); - load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); - - // initialize kokkos rrtmgp pool allocator - const size_t base_ref = 18000; - const size_t ncol = gas_concs.ncol; - const size_t nlay = gas_concs.nlay; - const size_t nlev = SCREAM_NUM_VERTICAL_LEV; - const size_t my_size_ref = ncol * nlay * nlev; - conv::MemPoolSingleton::init(2e6 * (float(my_size_ref) / base_ref)); - - // We are now initialized! - initialized_k = true; -} -#endif - void rrtmgp_finalize() { -#ifdef RRTMGP_ENABLE_YAKL initialized = false; k_dist_sw.finalize(); k_dist_lw.finalize(); cloud_optics_sw.finalize(); //~CloudOptics(); cloud_optics_lw.finalize(); //~CloudOptics(); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS - initialized_k = false; - k_dist_sw_k.finalize(); - k_dist_lw_k.finalize(); - cloud_optics_sw_k.finalize(); //~CloudOptics(); - cloud_optics_lw_k.finalize(); //~CloudOptics(); - conv::MemPoolSingleton::finalize(); -#endif } -#ifdef RRTMGP_ENABLE_YAKL void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, @@ -527,56 +293,7 @@ void compute_band_by_band_surface_albedos( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif) { - - EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); - auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); - - EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, - "Error! 1st dimension for wavenumber_limits should be 2. It's " << wavenumber_limits.extent(0)); - EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), - "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); - - // Loop over bands, and determine for each band whether it is broadly in the - // visible or infrared part of the spectrum (visible or "not visible") - Kokkos::parallel_for(conv::get_mdrp<2>({nswbands, ncol}), KOKKOS_LAMBDA(const int ibnd, const int icol) { - - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; - - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - - if (is_visible_wave1 && is_visible_wave2) { - // Entire band is in the visible - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); - } - else if (!is_visible_wave1 && !is_visible_wave2) { - // Entire band is in the longwave (near-infrared) - sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); - sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); - } - else { - // Band straddles the visible to near-infrared transition, so we take - // the albedo to be the average of the visible and near-infrared - // broadband albedos - sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); - sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , @@ -631,62 +348,7 @@ void compute_broadband_surface_fluxes( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , - real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, - real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir) { - // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums - // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when - // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This - // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. - //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); - //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); - - // Initialize sums over bands - Kokkos::deep_copy(sfc_flux_dir_nir, 0); - Kokkos::deep_copy(sfc_flux_dir_vis, 0); - Kokkos::deep_copy(sfc_flux_dif_nir, 0); - Kokkos::deep_copy(sfc_flux_dif_vis, 0); - - // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. - const real visible_wavenumber_threshold = 14286; - auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(const int icol) { - for (int ibnd = 0; ibnd < nswbands; ++ibnd) { - // Wavenumber is in the visible if it is above the visible wavenumber - // threshold, and in the infrared if it is below the threshold - const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); - const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); - - if (is_visible_wave1 && is_visible_wave2) { - // Entire band is in the visible - sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - } - else if (!is_visible_wave1 && !is_visible_wave2) { - // Entire band is in the longwave (near-infrared) - sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); - } - else { - // Band straddles the visible to near-infrared transition, so put half - // the flux in visible and half in near-infrared fluxes - sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); - sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); - } - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_main( const int ncol, const int nlay, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -855,191 +517,19 @@ void rrtmgp_main( ); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_main( - const int ncol, const int nlay, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, - real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, - real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, - real3dk &cld_tau_sw_gpt, - real3dk &cld_tau_lw_gpt, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn, - real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, - real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, - real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, - real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, - real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, - real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, - real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, - real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - -#ifdef SCREAM_RRTMGP_DEBUG - // Sanity check inputs, and possibly repair - check_range_k(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); - check_range_k(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); - check_range_k(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); - check_range_k(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); - check_range_k(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); - check_range_k(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); - check_range_k(mu0 , 0, 1, "rrtmgp_main::mu0"); - check_range_k(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); - check_range_k(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); - check_range_k(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); - check_range_k(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); -#endif - - // Setup pointers to RRTMGP SW fluxes - FluxesBybandK fluxes_sw; - fluxes_sw.flux_up = sw_flux_up; - fluxes_sw.flux_dn = sw_flux_dn; - fluxes_sw.flux_dn_dir = sw_flux_dn_dir; - fluxes_sw.bnd_flux_up = sw_bnd_flux_up; - fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; - fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; - // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_sw; - clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; - clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; - clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; - // Clear-sky - FluxesBroadbandK clrsky_fluxes_sw; - clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; - clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; - clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; - // Clean-sky - FluxesBroadbandK clnsky_fluxes_sw; - clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; - clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; - clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; - // Setup pointers to RRTMGP LW fluxes - FluxesBybandK fluxes_lw; - fluxes_lw.flux_up = lw_flux_up; - fluxes_lw.flux_dn = lw_flux_dn; - fluxes_lw.bnd_flux_up = lw_bnd_flux_up; - fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; - // Clean-clear-sky - FluxesBroadbandK clnclrsky_fluxes_lw; - clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; - clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; - // Clear-sky - FluxesBroadbandK clrsky_fluxes_lw; - clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; - clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; - // Clean-sky - FluxesBroadbandK clnsky_fluxes_lw; - clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; - clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; +int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { - auto nswbands = k_dist_sw_k.get_nband(); - auto nlwbands = k_dist_lw_k.get_nband(); + // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); - // Setup aerosol optical properties - OpticalProps2strK aerosol_sw; - OpticalProps1sclK aerosol_lw; - aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); - aerosol_sw.alloc_2str(ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); - aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); - aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); - }); - aerosol_lw.init(k_dist_lw_k.get_band_lims_wavenumber()); - aerosol_lw.alloc_1scl(ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { - aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); - }); - -#ifdef SCREAM_RRTMGP_DEBUG - // Check aerosol optical properties - // NOTE: these should already have been checked by precondition checks, but someday we might have - // non-trivial aerosol optics, so this is still good to do here. - check_range_k(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); - check_range_k(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); - check_range_k(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); - check_range_k(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); -#endif - - // Convert cloud physical properties to optical properties for input to RRTMGP - OpticalProps2strK clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); - OpticalProps1sclK clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); - Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); - Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); - - // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; - // This implements the Monte Carlo Independing Column Approximation by mapping only a single - // subcolumn (cloud state) to each gpoint. - auto nswgpts = k_dist_sw_k.get_ngpt(); - auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw_k, cldfrac, p_lay); - // Longwave - auto nlwgpts = k_dist_lw_k.get_ngpt(); - auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw_k, cldfrac, p_lay); - - // Copy cloud properties to outputs (is this needed, or can we just use pointers?) - // Alternatively, just compute and output a subcolumn cloud mask - Kokkos::parallel_for(conv::get_mdrp<3>({nswgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { - cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); - }); - -#ifdef SCREAM_RRTMGP_DEBUG - // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, - // but we might want to provide additional debug info here. NOTE: we may actually want to move this - // up higher in the code, I think optical props should go up higher since optical props are kind of - // a parameterization of their own, and we might want to swap different choices. These checks go here - // only because we need to run them on computed optical props, so if the optical props themselves get - // computed up higher, then perform these checks higher as well - check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); - check_range_k(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); - check_range_k(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); - check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); -#endif - - // Do shortwave - rrtmgp_sw( - ncol, nlay, - k_dist_sw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, - sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, - fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, - tsi_scaling, logger, - extra_clnclrsky_diag, extra_clnsky_diag - ); - - // Do longwave - rrtmgp_lw( - ncol, nlay, - k_dist_lw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, - aerosol_lw, clouds_lw_gpt, - fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, - extra_clnclrsky_diag, extra_clnsky_diag - ); - -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL -int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds) { - - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = int3d("subcolumn_mask", ncol, nlay, ngpt); - - // Subcolumn generators are a means for producing a variable x(i,j,k), where - // - // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) - // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) - // - // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = real3d("cldx", ncol, nlay, ngpt); + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = real3d("cldx", ncol, nlay, ngpt); // Apply overlap assumption to set cldx if (overlap_option == 0) { // Dummy mask, always cloudy @@ -1090,84 +580,7 @@ int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d }); return subcolumn_mask; } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds) { - - // Routine will return subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto subcolumn_mask = int3dk("subcolumn_mask", ncol, nlay, ngpt); - - // Subcolumn generators are a means for producing a variable x(i,j,k), where - // - // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) - // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) - // - // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = real3dk("cldx", ncol, nlay, ngpt); - - // Apply overlap assumption to set cldx - if (overlap_option == 0) { // Dummy mask, always cloudy - Kokkos::deep_copy(cldx, 1); - } else { // Default case, maximum-random overlap - // Maximum-random overlap: - // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, - // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same - // algorithm used in RRTMG implementation of maximum-random overlap (see - // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) - // - // First, fill cldx with random numbers. Need to use a unique seed for each column! - // auto seeds_host = Kokkos::create_mirror_view(seeds); - // Kokkos::deep_copy(seeds_host, seeds); - // for (int icol = 0; icol < ncol; ++icol) { - // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); - // Kokkos::parallel_for(conv::get_mdrp<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { - // auto generator = random_pool.get_state(); - // cldx(icol,ilay,igpt) = generator.drand(0., 1.); - // random_pool.free_state(generator); - // }); - // } - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - conv::Random rand(seeds(icol)); - for (int igpt = 0; igpt < ngpt; igpt++) { - for (int ilay = 0; ilay < nlay; ilay++) { - cldx(icol,ilay,igpt) = rand.genFP(); - } - } - }); - - // Step down columns and apply algorithm from eq (14) - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { - for (int ilay = 1; ilay < nlay; ilay++) { - // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn - if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { - // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent - // layers are maximimally overlapped - cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); - } else { - // Cloud-less above, use new random number so that clouds are distributed - // randomly in this layer. Need to scale new random number to range - // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution - // of random numbers in this layer with the above branch of the conditional, - // which would otherwise inflate cloud fraction in this layer. - cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); - } - } - }); - } - - // Use cldx array to create subcolumn mask - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { - subcolumn_mask(icol,ilay,igpt) = 1; - } else { - subcolumn_mask(icol,ilay,igpt) = 0; - } - }); - return subcolumn_mask; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_sw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, @@ -1423,267 +836,7 @@ void rrtmgp_sw( }); } } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_sw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Get problem sizes - int nbnd = k_dist.get_nband(); - int ngpt = k_dist.get_ngpt(); - int ngas = gas_concs.get_num_gases(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &flux_dn_dir = fluxes.flux_dn_dir; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; - - // Reset fluxes to zero - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilev, int icol) { - flux_up (icol,ilev) = 0; - flux_dn (icol,ilev) = 0; - flux_dn_dir(icol,ilev) = 0; - clnclrsky_flux_up (icol,ilev) = 0; - clnclrsky_flux_dn (icol,ilev) = 0; - clnclrsky_flux_dn_dir(icol,ilev) = 0; - clrsky_flux_up (icol,ilev) = 0; - clrsky_flux_dn (icol,ilev) = 0; - clrsky_flux_dn_dir(icol,ilev) = 0; - clnsky_flux_up (icol,ilev) = 0; - clnsky_flux_dn (icol,ilev) = 0; - clnsky_flux_dn_dir(icol,ilev) = 0; - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up (icol,ilev,ibnd) = 0; - bnd_flux_dn (icol,ilev,ibnd) = 0; - bnd_flux_dn_dir(icol,ilev,ibnd) = 0; - }); - - // Get daytime indices - auto dayIndices = int1dk("dayIndices", ncol); - Kokkos::deep_copy(dayIndices, -1); - - int nday = 0; - // Serialized for now. - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { - for (int icol = 0; icol < ncol; ++icol) { - if (mu0(icol) > 0) { - dayIndices(nday_inner++) = icol; - } - } - }, Kokkos::Sum(nday)); - - if (nday == 0) { - // No daytime columns in this chunk, skip the rest of this routine - return; - } - - // Subset mu0 - auto mu0_day = real1dk("mu0_day", nday); - Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { - mu0_day(iday) = mu0(dayIndices(iday)); - }); - - // subset state variables - auto p_lay_day = real2dk("p_lay_day", nday, nlay); - auto t_lay_day = real2dk("t_lay_day", nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { - p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); - t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); - }); - auto p_lev_day = real2dk("p_lev_day", nday, nlay+1); - auto t_lev_day = real2dk("t_lev_day", nday, nlay+1); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); - t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); - }); - - // Subset gases - auto gas_names = gas_concs.get_gas_names(); - GasConcsK gas_concs_day; - gas_concs_day.init(gas_names, nday, nlay); - for (int igas = 0; igas < ngas; igas++) { - auto vmr_day = real2dk("vmr_day", nday, nlay); - auto vmr = real2dk("vmr" , ncol, nlay); - gas_concs.get_vmr(gas_names[igas], vmr); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { - vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); - }); - gas_concs_day.set_vmr(gas_names[igas], vmr_day); - } - - // Subset aerosol optics - OpticalProps2strK aerosol_day; - aerosol_day.init(k_dist.get_band_lims_wavenumber()); - aerosol_day.alloc_2str(nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { - aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); - aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); - aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); - }); - - // Subset cloud optics - // TODO: nbnd -> ngpt once we pass sub-sampled cloud state - OpticalProps2strK clouds_day; - clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); - clouds_day.alloc_2str(nday, nlay); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { - clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); - clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); - clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); - }); - - // RRTMGP assumes surface albedos have a screwy dimension ordering - // for some strange reason, so we need to transpose these; also do - // daytime subsetting in the same kernel - real2dk sfc_alb_dir_T("sfc_alb_dir", nbnd, nday); - real2dk sfc_alb_dif_T("sfc_alb_dif", nbnd, nday); - Kokkos::parallel_for(conv::get_mdrp<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { - sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); - sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); - }); - - // Temporaries we need for daytime-only fluxes - auto flux_up_day = real2dk("flux_up_day", nday, nlay+1); - auto flux_dn_day = real2dk("flux_dn_day", nday, nlay+1); - auto flux_dn_dir_day = real2dk("flux_dn_dir_day", nday, nlay+1); - auto bnd_flux_up_day = real3dk("bnd_flux_up_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_day = real3dk("bnd_flux_dn_day", nday, nlay+1, nbnd); - auto bnd_flux_dn_dir_day = real3dk("bnd_flux_dn_dir_day", nday, nlay+1, nbnd); - FluxesBybandK fluxes_day; - fluxes_day.flux_up = flux_up_day; - fluxes_day.flux_dn = flux_dn_day; - fluxes_day.flux_dn_dir = flux_dn_dir_day; - fluxes_day.bnd_flux_up = bnd_flux_up_day; - fluxes_day.bnd_flux_dn = bnd_flux_dn_day; - fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; - - // Allocate space for optical properties - OpticalProps2strK optics; - optics.alloc_2str(nday, nlay, k_dist); - - OpticalProps2strK optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_2str(nday, nlay, k_dist); - } - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2dk("t_lay_limited", nday, nlay); - limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); - - // Do gas optics - real2dk toa_flux("toa_flux", nday, ngpt); - bool top_at_1 = false; - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { - val |= p_lay(0, 0) < p_lay(0, nlay-1); - }, Kokkos::LOr(top_at_1)); - - realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); - if (extra_clnsky_diag) { - k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); - } - -#ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); - check_range_k(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); - check_range_k(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); -#endif - - // Apply tsi_scaling - Kokkos::parallel_for(conv::get_mdrp<2>({ngpt,nday}), KOKKOS_LAMBDA(int igpt, int iday) { - toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); - }); - - if (extra_clnclrsky_diag) { - // Compute clear-clean-sky (just gas) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } - - // Combine gas and aerosol optics - aerosol_day.delta_scale(); - aerosol_day.increment(optics); - - // Compute clearsky (gas + aerosol) fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - - // Now merge in cloud optics and do allsky calculations - - // Combine gas and cloud optics - clouds_day.delta_scale(); - clouds_day.increment(optics); - // Compute fluxes on daytime columns - rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - flux_up (icol,ilev) = flux_up_day (iday,ilev); - flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,nday}), KOKKOS_LAMBDA(int ibnd, int ilev, int iday) { - const int icol = dayIndices(iday); - bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); - bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); - bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); - }); - - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds_day.increment(optics_no_aerosols); - // Compute cleansky (gas + clouds) fluxes on daytime columns - rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); - // Expand daytime fluxes to all columns - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { - const int icol = dayIndices(iday); - clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); - clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); - clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); - }); - } -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void rrtmgp_lw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, @@ -1816,147 +969,7 @@ void rrtmgp_lw( } } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, - OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { - - // Problem size - int nbnd = k_dist.get_nband(); - - // Associate local pointers for fluxes - auto &flux_up = fluxes.flux_up; - auto &flux_dn = fluxes.flux_dn; - auto &bnd_flux_up = fluxes.bnd_flux_up; - auto &bnd_flux_dn = fluxes.bnd_flux_dn; - auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; - auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; - auto &clrsky_flux_up = clrsky_fluxes.flux_up; - auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; - auto &clnsky_flux_up = clnsky_fluxes.flux_up; - auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; - - // Reset fluxes to zero - Kokkos::parallel_for( - conv::get_mdrp<2>({nlay + 1, ncol}), KOKKOS_LAMBDA(int ilev, int icol) { - flux_up(icol, ilev) = 0; - flux_dn(icol, ilev) = 0; - clnclrsky_flux_up(icol, ilev) = 0; - clnclrsky_flux_dn(icol, ilev) = 0; - clrsky_flux_up(icol, ilev) = 0; - clrsky_flux_dn(icol, ilev) = 0; - clnsky_flux_up(icol, ilev) = 0; - clnsky_flux_dn(icol, ilev) = 0; - }); - Kokkos::parallel_for( - conv::get_mdrp<3>({nbnd, nlay + 1, ncol}), - KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { - bnd_flux_up(icol, ilev, ibnd) = 0; - bnd_flux_dn(icol, ilev, ibnd) = 0; - }); - // Allocate space for optical properties - OpticalProps1sclK optics; - optics.alloc_1scl(ncol, nlay, k_dist); - OpticalProps1sclK optics_no_aerosols; - if (extra_clnsky_diag) { - // Allocate space for optical properties (no aerosols) - optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); - } - - // Boundary conditions - SourceFuncLWK lw_sources; - lw_sources.alloc(ncol, nlay, k_dist); - real1dk t_sfc ("t_sfc" ,ncol); - real2dk emis_sfc("emis_sfc",nbnd,ncol); - - bool top_at_1 = false; - Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { - val |= p_lay(0, 0) < p_lay(0, nlay-1); - }, Kokkos::LOr(top_at_1)); - - // Surface temperature - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - t_sfc(icol) = t_lev(icol, conv::merge(nlay, 0, top_at_1)); - }); - Kokkos::deep_copy(emis_sfc , 0.98); - - // Get Gaussian quadrature weights - // TODO: move this crap out of userland! - // Weights and angle secants for first order (k=1) Gaussian quadrature. - // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 - // after Abramowitz & Stegun 1972, page 921 - int constexpr max_gauss_pts = 4; - realHost2dk gauss_Ds_host ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - gauss_Ds_host(0,0) = 1.66 ; gauss_Ds_host(1,0) = 0.; gauss_Ds_host(2,0) = 0.; gauss_Ds_host(3,0) = 0.; - gauss_Ds_host(0,1) = 1.18350343; gauss_Ds_host(1,1) = 2.81649655; gauss_Ds_host(2,1) = 0.; gauss_Ds_host(3,1) = 0.; - gauss_Ds_host(0,2) = 1.09719858; gauss_Ds_host(1,2) = 1.69338507; gauss_Ds_host(2,2) = 4.70941630; gauss_Ds_host(3,2) = 0.; - gauss_Ds_host(0,3) = 1.06056257; gauss_Ds_host(1,3) = 1.38282560; gauss_Ds_host(2,3) = 2.40148179; gauss_Ds_host(3,3) = 7.15513024; - - realHost2dk gauss_wts_host("gauss_wts",max_gauss_pts,max_gauss_pts); - gauss_wts_host(0,0) = 0.5 ; gauss_wts_host(1,0) = 0. ; gauss_wts_host(2,0) = 0. ; gauss_wts_host(3,0) = 0. ; - gauss_wts_host(0,1) = 0.3180413817; gauss_wts_host(1,1) = 0.1819586183; gauss_wts_host(2,1) = 0. ; gauss_wts_host(3,1) = 0. ; - gauss_wts_host(0,2) = 0.2009319137; gauss_wts_host(1,2) = 0.2292411064; gauss_wts_host(2,2) = 0.0698269799; gauss_wts_host(3,2) = 0. ; - gauss_wts_host(0,3) = 0.1355069134; gauss_wts_host(1,3) = 0.2034645680; gauss_wts_host(2,3) = 0.1298475476; gauss_wts_host(3,3) = 0.0311809710; - - real2dk gauss_Ds ("gauss_Ds" ,max_gauss_pts,max_gauss_pts); - real2dk gauss_wts("gauss_wts",max_gauss_pts,max_gauss_pts); - Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); - Kokkos::deep_copy(gauss_wts, gauss_wts_host); - - // Limit temperatures for gas optics look-up tables - auto t_lay_limited = real2dk("t_lay_limited", ncol, nlay); - auto t_lev_limited = real2dk("t_lev_limited", ncol, nlay+1); - limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); - limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); - - // Do gas optics - realOff3dk col_gas("col_gas", std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, real2dk(), t_lev_limited); - if (extra_clnsky_diag) { - k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, real2dk(), t_lev_limited); - } - -#ifdef SCREAM_RRTMGP_DEBUG - // Check gas optics - check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); -#endif - - if (extra_clnclrsky_diag) { - // Compute clean-clear-sky fluxes before we add in aerosols and clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); - } - - // Combine gas and aerosol optics - aerosol.increment(optics); - - // Compute clear-sky fluxes before we add in clouds - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); - - // Combine gas and cloud optics - clouds.increment(optics); - - // Compute allsky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); - - if (extra_clnsky_diag) { - // First increment clouds in optics_no_aerosols - clouds.increment(optics_no_aerosols); - // Compute clean-sky fluxes - rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); - } - -} -#endif - -#ifdef RRTMGP_ENABLE_YAKL void compute_cloud_area( int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area) { @@ -1981,46 +994,11 @@ void compute_cloud_area( } }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_cloud_area( - int ncol, int nlay, int ngpt, const Real pmin, const Real pmax, - const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area) { - // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy - // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = real2dk("subcol_mask", ncol, ngpt); - Kokkos::parallel_for(conv::get_mdrp<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { - // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but - // using play/pmid does not - if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { - subcol_mask(icol,igpt) = 1; - } - }); - // Compute average over subcols to get cloud area - auto ngpt_inv = 1.0 / ngpt; - Kokkos::deep_copy(cld_area, 0); - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - // This loop needs to be serial because of the atomic reduction - for (int igpt = 0; igpt < ngpt; ++igpt) { - cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; - } - }); -} -#endif -#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index_sw(double wavelength) { return get_wavelength_index(k_dist_sw, wavelength); } int get_wavelength_index_lw(double wavelength) { return get_wavelength_index(k_dist_lw, wavelength); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index_sw_k(double wavelength) { return get_wavelength_index(k_dist_sw_k, wavelength); } - -int get_wavelength_index_lw_k(double wavelength) { return get_wavelength_index(k_dist_lw_k, wavelength); } -#endif - -#ifdef RRTMGP_ENABLE_YAKL int get_wavelength_index(OpticalProps &kdist, double wavelength) { // Get wavelength bounds for all wavelength bands auto wavelength_bounds = kdist.get_band_lims_wavelength(); @@ -2043,33 +1021,7 @@ int get_wavelength_index(OpticalProps &kdist, double wavelength) { }); return band_index.hostRead(); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index(OpticalPropsK &kdist, double wavelength) { - // Get wavelength bounds for all wavelength bands - auto wavelength_bounds = kdist.get_band_lims_wavelength(); - - // Find the band index for the specified wavelength - // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength - // in units of meters, we need a conversion factor of 10^2 - const int nbnds = kdist.get_nband(); - int band_index = -1; - Kokkos::parallel_reduce(nbnds, KOKKOS_LAMBDA(int ibnd, int& band_index_inner) { - if (wavelength_bounds(0,ibnd) < wavelength_bounds(1,ibnd)) { - if (wavelength_bounds(0,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(1,ibnd)) { - band_index_inner = ibnd; - } - } else { - if (wavelength_bounds(0,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(1,ibnd)) { - band_index_inner = ibnd; - } - } - }, Kokkos::Max(band_index)); - return band_index; -} -#endif -#ifdef RRTMGP_ENABLE_YAKL void compute_aerocom_cloudtop( int ncol, int nlay, const real2d &tmid, const real2d &pmid, const real2d &p_del, const real2d &z_del, const real2d &qc, @@ -2166,109 +1118,7 @@ void compute_aerocom_cloudtop( cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_aerocom_cloudtop( - int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, - const real2dk &p_del, const real2dk &z_del, const real2dk &qc, - const real2dk &qi, const real2dk &rel, const real2dk &rei, - const real2dk &cldfrac_tot, const real2dk &nc, - real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, - real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, - real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, - real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop) { - /* The goal of this routine is to calculate properties at cloud top - * based on the AeroCom recommendation. See reference for routine - * get_subcolumn_mask above, where equation 14 is used for the - * maximum-random overlap assumption for subcolumn generation. We use - * equation 13, the column counterpart. - */ - // Set outputs to zero - Kokkos::deep_copy(T_mid_at_cldtop, 0.0); - Kokkos::deep_copy(p_mid_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_ice_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_liq_at_cldtop, 0.0); - Kokkos::deep_copy(cldfrac_tot_at_cldtop, 0.0); - Kokkos::deep_copy(cdnc_at_cldtop, 0.0); - Kokkos::deep_copy(eff_radius_qc_at_cldtop, 0.0); - Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); - - // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = real1dk("aerocom_clr", ncol); - Kokkos::deep_copy(aerocom_clr, 1.0); - - // Get gravity acceleration constant from constants - using physconst = scream::physics::Constants; - - // TODO: move tunable constant to namelist - constexpr real q_threshold = 0.0; // BAD_CONSTANT! - - // TODO: move tunable constant to namelist - constexpr real cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! - - // Loop over all columns in parallel - Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { - // Loop over all layers in serial (due to accumulative - // product), starting at 2 (second highest) layer because the - // highest is assumed to hav no clouds - for(int ilay = 1; ilay < nlay; ++ilay) { - // Only do the calculation if certain conditions are met - if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && - (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { - /* PART I: Probabilistically determining cloud top */ - // Populate aerocom_tmp as the clear-sky fraction - // probability of this level, where aerocom_clr is that of - // the previous level - auto aerocom_tmp = - aerocom_clr(icol) * - (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), - cldfrac_tot(icol, ilay))) / - (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), - 1.0 - cldfrac_tot_threshold)); - // Temporary variable for probability "weights" - auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; - // Temporary variable for liquid "phase" - auto aerocom_phi = - qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); - /* PART II: The inferred properties */ - /* In general, converting a 3D property X to a 2D cloud-top - * counterpart x follows: x(i) += X(i,k) * weights * Phase - * but X and Phase are not always needed */ - // T_mid_at_cldtop - T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; - // p_mid_at_cldtop - p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; - // cldfrac_ice_at_cldtop - cldfrac_ice_at_cldtop(icol) += - (1.0 - aerocom_phi) * aerocom_wts; - // cldfrac_liq_at_cldtop - cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; - // cdnc_at_cldtop - /* We need to convert nc from 1/mass to 1/volume first, and - * from grid-mean to in-cloud, but after that, the - * calculation follows the general logic */ - auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / - z_del(icol, ilay) / physconst::gravit / - cldfrac_tot(icol, ilay); - cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; - // eff_radius_qc_at_cldtop - eff_radius_qc_at_cldtop(icol) += - rel(icol, ilay) * aerocom_phi * aerocom_wts; - // eff_radius_qi_at_cldtop - eff_radius_qi_at_cldtop(icol) += - rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; - // Reset aerocom_clr to aerocom_tmp to accumulate - aerocom_clr(icol) = aerocom_tmp; - } - } - // After the serial loop over levels, the cloudy fraction is - // defined as (1 - aerocom_clr). This is true because - // aerocom_clr is the result of accumulative probabilities - // (their products) - cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); - }); -} -#endif } // namespace rrtmgp +#endif } // namespace scream diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index 49fbfeb4486b..a420c0c1dd4f 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -4,12 +4,26 @@ #include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" #include "cpp/extensions/cloud_optics/mo_cloud_optics.h" #include "cpp/extensions/fluxes_byband/mo_fluxes_byband.h" +#include "cpp/examples/mo_load_coefficients.h" #include "cpp/rrtmgp_const.h" +#include "cpp/rrtmgp/mo_gas_concentrations.h" +#include "cpp/rrtmgp/mo_gas_optics_rrtmgp.h" +#include "cpp/extensions/cloud_optics/mo_cloud_optics.h" +#include "cpp/rte/mo_rte_sw.h" +#include "cpp/rte/mo_rte_lw.h" +#include "examples/all-sky/mo_load_cloud_coefficients.h" + +#include "rrtmgp_utils.hpp" #include "physics/share/physics_constants.hpp" #include "ekat/mpi/ekat_comm.hpp" #include "ekat/logging/ekat_logger.hpp" +#include "ekat/util/ekat_math_utils.hpp" + +#ifdef RRTMGP_ENABLE_KOKKOS +#include "Kokkos_Random.hpp" +#endif namespace scream { @@ -18,105 +32,34 @@ void finalize_kls(); namespace rrtmgp { -/* - * Objects containing k-distribution information need to be initialized - * once and then persist throughout the life of the program, so we - * declare them here within the rrtmgp namespace. - */ #ifdef RRTMGP_ENABLE_YAKL extern GasOpticsRRTMGP k_dist_sw; extern GasOpticsRRTMGP k_dist_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern GasOpticsRRTMGPK k_dist_sw_k; -extern GasOpticsRRTMGPK k_dist_lw_k; -#endif -/* - * Objects containing cloud optical property look-up table information. - * We want to initialize these once and use throughout the life of the - * program, so declare here and read data in during rrtmgp_initialize(). - */ -#ifdef RRTMGP_ENABLE_YAKL extern CloudOptics cloud_optics_sw; extern CloudOptics cloud_optics_lw; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern CloudOpticsK cloud_optics_sw_k; -extern CloudOpticsK cloud_optics_lw_k; -#endif -/* - * Flag to indicate whether or not we have initialized RRTMGP - */ -#ifdef RRTMGP_ENABLE_YAKL extern bool initialized; -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern bool initialized_k; -#endif -/* - * Initialize data for RRTMGP driver - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_initialize( +void rrtmgp_initialize( GasConcs &gas_concs, const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, const std::shared_ptr& logger); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_initialize( - GasConcsK &gas_concs, - const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, - const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, - const std::shared_ptr& logger); -#endif -/* - * Compute band-by-band surface albedos from broadband albedos. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void compute_band_by_band_surface_albedos( +void compute_band_by_band_surface_albedos( const int ncol, const int nswbands, real1d &sfc_alb_dir_vis, real1d &sfc_alb_dir_nir, real1d &sfc_alb_dif_vis, real1d &sfc_alb_dif_nir, real2d &sfc_alb_dir, real2d &sfc_alb_dif); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void compute_band_by_band_surface_albedos( - const int ncol, const int nswbands, - real1dk &sfc_alb_dir_vis, real1dk &sfc_alb_dir_nir, - real1dk &sfc_alb_dif_vis, real1dk &sfc_alb_dif_nir, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif); -#endif -/* - * Compute broadband visible/UV and near-infrared surface fluxes. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void compute_broadband_surface_fluxes( +void compute_broadband_surface_fluxes( const int ncol, const int ktop, const int nswbands, real3d &sw_bnd_flux_dir , real3d &sw_bnd_flux_dif , real1d &sfc_flux_dir_vis, real1d &sfc_flux_dir_nir, real1d &sfc_flux_dif_vis, real1d &sfc_flux_dif_nir); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void compute_broadband_surface_fluxes( - const int ncol, const int ktop, const int nswbands, - real3dk &sw_bnd_flux_dir , real3dk &sw_bnd_flux_dif , - real1dk &sfc_flux_dir_vis, real1dk &sfc_flux_dir_nir, - real1dk &sfc_flux_dif_vis, real1dk &sfc_flux_dif_nir); -#endif -/* - * Main driver code to run RRTMGP. - * The input logger is in charge of outputing info to - * screen and/or to file (or neither), depending on how it was set up. - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_main( +void rrtmgp_main( const int ncol, const int nlay, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, GasConcs &gas_concs, @@ -138,42 +81,10 @@ extern void rrtmgp_main( const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_main( - const int ncol, const int nlay, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - real2dk &lwp, real2dk &iwp, real2dk &rel, real2dk &rei, real2dk &cldfrac, - real3dk &aer_tau_sw, real3dk &aer_ssa_sw, real3dk &aer_asm_sw, real3dk &aer_tau_lw, - real3dk &cld_tau_sw_bnd, real3dk &cld_tau_lw_bnd, - real3dk &cld_tau_sw_gpt, real3dk &cld_tau_lw_gpt, - real2dk &sw_flux_up, real2dk &sw_flux_dn, real2dk &sw_flux_dn_dir, - real2dk &lw_flux_up, real2dk &lw_flux_dn, - real2dk &sw_clnclrsky_flux_up, real2dk &sw_clnclrsky_flux_dn, real2dk &sw_clnclrsky_flux_dn_dir, - real2dk &sw_clrsky_flux_up, real2dk &sw_clrsky_flux_dn, real2dk &sw_clrsky_flux_dn_dir, - real2dk &sw_clnsky_flux_up, real2dk &sw_clnsky_flux_dn, real2dk &sw_clnsky_flux_dn_dir, - real2dk &lw_clnclrsky_flux_up, real2dk &lw_clnclrsky_flux_dn, - real2dk &lw_clrsky_flux_up, real2dk &lw_clrsky_flux_dn, - real2dk &lw_clnsky_flux_up, real2dk &lw_clnsky_flux_dn, - real3dk &sw_bnd_flux_up, real3dk &sw_bnd_flux_dn, real3dk &sw_bnd_flux_dn_dir, - real3dk &lw_bnd_flux_up, real3dk &lw_bnd_flux_dn, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false); -#endif -/* - * Perform any clean-up tasks - */ -extern void rrtmgp_finalize(); +void rrtmgp_finalize(); -/* - * Shortwave driver (called by rrtmgp_main) - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_sw( +void rrtmgp_sw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -184,26 +95,8 @@ extern void rrtmgp_sw( const Real tsi_scaling, const std::shared_ptr& logger, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_sw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - real2dk &sfc_alb_dir, real2dk &sfc_alb_dif, real1dk &mu0, - OpticalProps2strK &aerosol, OpticalProps2strK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const Real tsi_scaling, - const std::shared_ptr& logger, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -/* - * Longwave driver (called by rrtmgp_main) - */ -#ifdef RRTMGP_ENABLE_YAKL -extern void rrtmgp_lw( +void rrtmgp_lw( const int ncol, const int nlay, GasOpticsRRTMGP &k_dist, real2d &p_lay, real2d &t_lay, real2d &p_lev, real2d &t_lev, @@ -211,46 +104,13 @@ extern void rrtmgp_lw( OpticalProps1scl &aerosol, OpticalProps1scl &clouds, FluxesByband &fluxes, FluxesBroadband &clnclrsky_fluxes, FluxesBroadband &clrsky_fluxes, FluxesBroadband &clnsky_fluxes, const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -extern void rrtmgp_lw( - const int ncol, const int nlay, - GasOpticsRRTMGPK &k_dist, - real2dk &p_lay, real2dk &t_lay, real2dk &p_lev, real2dk &t_lev, - GasConcsK &gas_concs, - OpticalProps1sclK &aerosol, OpticalProps1sclK &clouds, - FluxesBybandK &fluxes, FluxesBroadbandK &clnclrsky_fluxes, FluxesBroadbandK &clrsky_fluxes, FluxesBroadbandK &clnsky_fluxes, - const bool extra_clnclrsky_diag, const bool extra_clnsky_diag); -#endif -/* - * Return a subcolumn mask consistent with a specified overlap assumption - */ -#ifdef RRTMGP_ENABLE_YAKL int3d get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2d &cldf, const int overlap_option, int1d &seeds); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int3dk get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, real2dk &cldf, const int overlap_option, int1dk &seeds); -#endif -/* - * Compute cloud area from 3d subcol cloud property - */ -#ifdef RRTMGP_ENABLE_YAKL void compute_cloud_area( int ncol, int nlay, int ngpt, Real pmin, Real pmax, const real2d& pmid, const real3d& cld_tau_gpt, real1d& cld_area); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_cloud_area( - int ncol, int nlay, int ngpt, Real pmin, Real pmax, - const real2dk& pmid, const real3dk& cld_tau_gpt, real1dk& cld_area); -#endif -/* - * Return select cloud-top diagnostics following AeroCom recommendation - */ -#ifdef RRTMGP_ENABLE_YAKL void compute_aerocom_cloudtop( int ncol, int nlay, const real2d &tmid, const real2d &pmid, const real2d &p_del, const real2d &z_del, const real2d &qc, @@ -260,26 +120,7 @@ void compute_aerocom_cloudtop( real1d &cldfrac_ice_at_cldtop, real1d &cldfrac_liq_at_cldtop, real1d &cldfrac_tot_at_cldtop, real1d &cdnc_at_cldtop, real1d &eff_radius_qc_at_cldtop, real1d &eff_radius_qi_at_cldtop); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -void compute_aerocom_cloudtop( - int ncol, int nlay, const real2dk &tmid, const real2dk &pmid, - const real2dk &p_del, const real2dk &z_del, const real2dk &qc, - const real2dk &qi, const real2dk &rel, const real2dk &rei, - const real2dk &cldfrac_tot, const real2dk &nc, - real1dk &T_mid_at_cldtop, real1dk &p_mid_at_cldtop, - real1dk &cldfrac_ice_at_cldtop, real1dk &cldfrac_liq_at_cldtop, - real1dk &cldfrac_tot_at_cldtop, real1dk &cdnc_at_cldtop, - real1dk &eff_radius_qc_at_cldtop, real1dk &eff_radius_qi_at_cldtop); -#endif -/* - * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area - * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion - * with the standard practice of using "water path" to refer to the total column-integrated - * quantities). - */ -#ifdef RRTMGP_ENABLE_YAKL template void mixing_ratio_to_cloud_mass( yakl::Array const &mixing_ratio, @@ -303,10 +144,1065 @@ void mixing_ratio_to_cloud_mass( } }); } -#endif + +template +void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { + yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { + arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + }); +} + +int get_wavelength_index(OpticalProps &kdist, double wavelength); +int get_wavelength_index_sw(double wavelength); +int get_wavelength_index_lw(double wavelength); +#endif // RRTMGP_ENABLE_YAKL + +// New interface for Kokkos and flexible types #ifdef RRTMGP_ENABLE_KOKKOS +template +struct rrtmgp_interface { + +using MDRP = typename conv::MDRP; + +template +using view_t = Kokkos::View; + +template +using oview_t = Kokkos::Experimental::OffsetView; + +template +using hview_t = Kokkos::View; + +using pool_t = conv::MemPoolSingleton; + +using real1dk = view_t; +using real2dk = view_t; +using real3dk = view_t; +using creal1dk = view_t; +using creal2dk = view_t; +using creal3dk = view_t; +using int1dk = view_t; +using int3dk = view_t; + +using gas_optics_t = GasOpticsRRTMGPK; +using cloud_optics_t = CloudOpticsK; +using gas_concs_t = GasConcsK; +using fluxes_t = FluxesBybandK; +using fluxes_broadband_t = FluxesBroadbandK; +using optical_props_t = OpticalPropsK; +using optical_props1_t = OpticalProps1sclK; +using optical_props2_t = OpticalProps2strK; +using source_func_t = SourceFuncLWK; + +/* + * Objects containing k-distribution information need to be initialized + * once and then persist throughout the life of the program, so we + * declare them here within the rrtmgp namespace. + */ +static inline gas_optics_t k_dist_sw_k; +static inline gas_optics_t k_dist_lw_k; + +/* + * Objects containing cloud optical property look-up table information. + * We want to initialize these once and use throughout the life of the + * program, so declare here and read data in during rrtmgp_initialize(). + */ +static inline cloud_optics_t cloud_optics_sw_k; +static inline cloud_optics_t cloud_optics_lw_k; + +/* + * Flag to indicate whether or not we have initialized RRTMGP + */ +static inline bool initialized_k = false; + +/* + * Initialize data for RRTMGP driver + */ +static void rrtmgp_initialize( + const gas_concs_t &gas_concs, + const std::string& coefficients_file_sw, const std::string& coefficients_file_lw, + const std::string& cloud_optics_file_sw, const std::string& cloud_optics_file_lw, + const std::shared_ptr& logger) +{ + // If we've already initialized, just exit + if (initialized_k) { + if (logger) + logger->info("RRTMGP is already initialized; skipping\n"); + return; + } + + // Initialize Kokkos + if (!Kokkos::is_initialized()) { Kokkos::initialize(); } + + // Load and initialize absorption coefficient data + load_and_init(k_dist_sw_k, coefficients_file_sw, gas_concs); + load_and_init(k_dist_lw_k, coefficients_file_lw, gas_concs); + + // Load and initialize cloud optical property look-up table information + load_cld_lutcoeff(cloud_optics_sw_k, cloud_optics_file_sw); + load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); + + // initialize kokkos rrtmgp pool allocator + const size_t base_ref = 18000; + const size_t ncol = gas_concs.ncol; + const size_t nlay = gas_concs.nlay; + const size_t nlev = SCREAM_NUM_VERTICAL_LEV; + const size_t my_size_ref = ncol * nlay * nlev; + pool_t::init(2e6 * (float(my_size_ref) / base_ref)); + + // We are now initialized! + initialized_k = true; +} + +/* + * Compute band-by-band surface albedos from broadband albedos. + */ +static void compute_band_by_band_surface_albedos( + const int ncol, const int nswbands, + const creal1dk &sfc_alb_dir_vis, const creal1dk &sfc_alb_dir_nir, + const creal1dk &sfc_alb_dif_vis, const creal1dk &sfc_alb_dif_nir, + const real2dk &sfc_alb_dir, const real2dk &sfc_alb_dif) +{ + EKAT_ASSERT_MSG(initialized_k, "Error! rrtmgp_initialize must be called before GasOpticsRRTMGP object can be used."); + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + + EKAT_ASSERT_MSG(wavenumber_limits.extent(0) == 2, + "Error! 1st dimension for wavenumber_limits should be 2. It's " << wavenumber_limits.extent(0)); + EKAT_ASSERT_MSG(wavenumber_limits.extent(1) == static_cast(nswbands), + "Error! 2nd dimension for wavenumber_limits should be " + std::to_string(nswbands) + " (nswbands)."); + + // Loop over bands, and determine for each band whether it is broadly in the + // visible or infrared part of the spectrum (visible or "not visible") + Kokkos::parallel_for(MDRP::template get<2>({nswbands, ncol}), KOKKOS_LAMBDA(const int ibnd, const int icol) { + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const RealT visible_wavenumber_threshold = 14286; + + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_vis(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_vis(icol); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_alb_dir(icol,ibnd) = sfc_alb_dir_nir(icol); + sfc_alb_dif(icol,ibnd) = sfc_alb_dif_nir(icol); + } + else { + // Band straddles the visible to near-infrared transition, so we take + // the albedo to be the average of the visible and near-infrared + // broadband albedos + sfc_alb_dir(icol,ibnd) = 0.5*(sfc_alb_dir_vis(icol) + sfc_alb_dir_nir(icol)); + sfc_alb_dif(icol,ibnd) = 0.5*(sfc_alb_dif_vis(icol) + sfc_alb_dif_nir(icol)); + } + }); +} + +/* + * Compute broadband visible/UV and near-infrared surface fluxes. + */ +static void compute_broadband_surface_fluxes( + const int ncol, const int ktop, const int nswbands, + const real3dk &sw_bnd_flux_dir , const real3dk &sw_bnd_flux_dif , + const real1dk &sfc_flux_dir_vis, const real1dk &sfc_flux_dir_nir, + const real1dk &sfc_flux_dif_vis, const real1dk &sfc_flux_dif_nir) +{ + // Band 10 straddles the near-IR and visible, so divide contributions from band 10 between both broadband sums + // TODO: Hard-coding these band indices is really bad practice. If the bands ever were to change (like when + // the RRTMG bands were re-ordered for RRTMGP), we would be using the wrong bands for the IR and UV/VIS. This + // should be refactored to grab the correct bands by specifying appropriate wavenumber rather than index. + //sfc_flux_dir_nir(i) = sum(sw_bnd_flux_dir(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dir_vis(i) = sum(sw_bnd_flux_dir(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dir(i+1,kbot,10); + //sfc_flux_dif_nir(i) = sum(sw_bnd_flux_dif(i+1,kbot,1:9)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + //sfc_flux_dif_vis(i) = sum(sw_bnd_flux_dif(i+1,kbot,11:14)) + 0.5 * sw_bnd_flux_dif(i+1,kbot,10); + + // Initialize sums over bands + Kokkos::deep_copy(sfc_flux_dir_nir, 0); + Kokkos::deep_copy(sfc_flux_dir_vis, 0); + Kokkos::deep_copy(sfc_flux_dif_nir, 0); + Kokkos::deep_copy(sfc_flux_dif_vis, 0); + + // Threshold between visible and infrared is 0.7 micron, or 14286 cm^-1. + const RealT visible_wavenumber_threshold = 14286; + auto wavenumber_limits = k_dist_sw_k.get_band_lims_wavenumber(); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(const int icol) { + for (int ibnd = 0; ibnd < nswbands; ++ibnd) { + // Wavenumber is in the visible if it is above the visible wavenumber + // threshold, and in the infrared if it is below the threshold + const bool is_visible_wave1 = (wavenumber_limits(0, ibnd) > visible_wavenumber_threshold ? true : false); + const bool is_visible_wave2 = (wavenumber_limits(1, ibnd) > visible_wavenumber_threshold ? true : false); + + if (is_visible_wave1 && is_visible_wave2) { + // Entire band is in the visible + sfc_flux_dir_vis(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else if (!is_visible_wave1 && !is_visible_wave2) { + // Entire band is in the longwave (near-infrared) + sfc_flux_dir_nir(icol) += sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += sw_bnd_flux_dif(icol,ktop,ibnd); + } + else { + // Band straddles the visible to near-infrared transition, so put half + // the flux in visible and half in near-infrared fluxes + sfc_flux_dir_vis(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_vis(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + sfc_flux_dir_nir(icol) += 0.5 * sw_bnd_flux_dir(icol,ktop,ibnd); + sfc_flux_dif_nir(icol) += 0.5 * sw_bnd_flux_dif(icol,ktop,ibnd); + } + } + }); +} + +/* + * Main driver code to run RRTMGP. + * The input logger is in charge of outputing info to + * screen and/or to file (or neither), depending on how it was set up. + */ +static void rrtmgp_main( + const int ncol, const int nlay, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + const creal2dk &sfc_alb_dir, const creal2dk &sfc_alb_dif, const real1dk &mu0, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei, const real2dk &cldfrac, + const real3dk &aer_tau_sw, const real3dk &aer_ssa_sw, const real3dk &aer_asm_sw, const real3dk &aer_tau_lw, + const real3dk &cld_tau_sw_bnd, const real3dk &cld_tau_lw_bnd, + const real3dk &cld_tau_sw_gpt, const real3dk &cld_tau_lw_gpt, + const real2dk &sw_flux_up, const real2dk &sw_flux_dn, const real2dk &sw_flux_dn_dir, + const real2dk &lw_flux_up, const real2dk &lw_flux_dn, + const real2dk &sw_clnclrsky_flux_up, const real2dk &sw_clnclrsky_flux_dn, const real2dk &sw_clnclrsky_flux_dn_dir, + const real2dk &sw_clrsky_flux_up, const real2dk &sw_clrsky_flux_dn, const real2dk &sw_clrsky_flux_dn_dir, + const real2dk &sw_clnsky_flux_up, const real2dk &sw_clnsky_flux_dn, const real2dk &sw_clnsky_flux_dn_dir, + const real2dk &lw_clnclrsky_flux_up, const real2dk &lw_clnclrsky_flux_dn, + const real2dk &lw_clrsky_flux_up, const real2dk &lw_clrsky_flux_dn, + const real2dk &lw_clnsky_flux_up, const real2dk &lw_clnsky_flux_dn, + const real3dk &sw_bnd_flux_up, const real3dk &sw_bnd_flux_dn, const real3dk &sw_bnd_flux_dn_dir, + const real3dk &lw_bnd_flux_up, const real3dk &lw_bnd_flux_dn, + const Real tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag = false, const bool extra_clnsky_diag = false) +{ +#ifdef SCREAM_RRTMGP_DEBUG + // Sanity check inputs, and possibly repair + check_range_k(t_lay , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lay"); + check_range_k(t_lev , k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), "rrtmgp_main::t_lev"); + check_range_k(p_lay , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lay"); + check_range_k(p_lev , k_dist_sw_k.get_press_min(), k_dist_sw_k.get_press_max(), "rrtmgp_main::p_lev"); + check_range_k(sfc_alb_dir, 0, 1, "rrtmgp_main::sfc_alb_dir"); + check_range_k(sfc_alb_dif, 0, 1, "rrtmgp_main::sfc_alb_dif"); + check_range_k(mu0 , 0, 1, "rrtmgp_main::mu0"); + check_range_k(lwp , 0, std::numeric_limits::max(), "rrtmgp_main::lwp"); + check_range_k(iwp , 0, std::numeric_limits::max(), "rrtmgp_main::iwp"); + check_range_k(rel , 0, std::numeric_limits::max(), "rrtmgp_main::rel"); + check_range_k(rei , 0, std::numeric_limits::max(), "rrtmgp_main::rei"); +#endif + + // Setup pointers to RRTMGP SW fluxes + fluxes_t fluxes_sw; + fluxes_sw.flux_up = sw_flux_up; + fluxes_sw.flux_dn = sw_flux_dn; + fluxes_sw.flux_dn_dir = sw_flux_dn_dir; + fluxes_sw.bnd_flux_up = sw_bnd_flux_up; + fluxes_sw.bnd_flux_dn = sw_bnd_flux_dn; + fluxes_sw.bnd_flux_dn_dir = sw_bnd_flux_dn_dir; + // Clean-clear-sky + fluxes_broadband_t clnclrsky_fluxes_sw; + clnclrsky_fluxes_sw.flux_up = sw_clnclrsky_flux_up; + clnclrsky_fluxes_sw.flux_dn = sw_clnclrsky_flux_dn; + clnclrsky_fluxes_sw.flux_dn_dir = sw_clnclrsky_flux_dn_dir; + // Clear-sky + fluxes_broadband_t clrsky_fluxes_sw; + clrsky_fluxes_sw.flux_up = sw_clrsky_flux_up; + clrsky_fluxes_sw.flux_dn = sw_clrsky_flux_dn; + clrsky_fluxes_sw.flux_dn_dir = sw_clrsky_flux_dn_dir; + // Clean-sky + fluxes_broadband_t clnsky_fluxes_sw; + clnsky_fluxes_sw.flux_up = sw_clnsky_flux_up; + clnsky_fluxes_sw.flux_dn = sw_clnsky_flux_dn; + clnsky_fluxes_sw.flux_dn_dir = sw_clnsky_flux_dn_dir; + + // Setup pointers to RRTMGP LW fluxes + fluxes_t fluxes_lw; + fluxes_lw.flux_up = lw_flux_up; + fluxes_lw.flux_dn = lw_flux_dn; + fluxes_lw.bnd_flux_up = lw_bnd_flux_up; + fluxes_lw.bnd_flux_dn = lw_bnd_flux_dn; + // Clean-clear-sky + fluxes_broadband_t clnclrsky_fluxes_lw; + clnclrsky_fluxes_lw.flux_up = lw_clnclrsky_flux_up; + clnclrsky_fluxes_lw.flux_dn = lw_clnclrsky_flux_dn; + // Clear-sky + fluxes_broadband_t clrsky_fluxes_lw; + clrsky_fluxes_lw.flux_up = lw_clrsky_flux_up; + clrsky_fluxes_lw.flux_dn = lw_clrsky_flux_dn; + // Clean-sky + fluxes_broadband_t clnsky_fluxes_lw; + clnsky_fluxes_lw.flux_up = lw_clnsky_flux_up; + clnsky_fluxes_lw.flux_dn = lw_clnsky_flux_dn; + + auto nswbands = k_dist_sw_k.get_nband(); + auto nlwbands = k_dist_lw_k.get_nband(); + + // Setup aerosol optical properties + optical_props2_t aerosol_sw; + optical_props1_t aerosol_lw; + aerosol_sw.init(k_dist_sw_k.get_band_lims_wavenumber()); + aerosol_sw.alloc_2str(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_sw.tau(icol,ilay,ibnd) = aer_tau_sw(icol,ilay,ibnd); + aerosol_sw.ssa(icol,ilay,ibnd) = aer_ssa_sw(icol,ilay,ibnd); + aerosol_sw.g (icol,ilay,ibnd) = aer_asm_sw(icol,ilay,ibnd); + }); + aerosol_lw.init(k_dist_lw_k.get_band_lims_wavenumber()); + aerosol_lw.alloc_1scl(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nlwbands,nlay,ncol}) , KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + aerosol_lw.tau(icol,ilay,ibnd) = aer_tau_lw(icol,ilay,ibnd); + }); + +#ifdef SCREAM_RRTMGP_DEBUG + // Check aerosol optical properties + // NOTE: these should already have been checked by precondition checks, but someday we might have + // non-trivial aerosol optics, so this is still good to do here. + check_range_k(aerosol_sw.tau, 0, 1e3, "rrtmgp_main:aerosol_sw.tau"); + check_range_k(aerosol_sw.ssa, 0, 1, "rrtmgp_main:aerosol_sw.ssa"); //, "aerosol_optics_sw.ssa"); + check_range_k(aerosol_sw.g , -1, 1, "rrtmgp_main:aerosol_sw.g "); //, "aerosol_optics_sw.g" ); + check_range_k(aerosol_lw.tau, 0, 1e3, "rrtmgp_main:aerosol_lw.tau"); +#endif + + // Convert cloud physical properties to optical properties for input to RRTMGP + optical_props2_t clouds_sw = get_cloud_optics_sw(ncol, nlay, cloud_optics_sw_k, k_dist_sw_k, lwp, iwp, rel, rei); + optical_props1_t clouds_lw = get_cloud_optics_lw(ncol, nlay, cloud_optics_lw_k, k_dist_lw_k, lwp, iwp, rel, rei); + Kokkos::deep_copy(cld_tau_sw_bnd, clouds_sw.tau); + Kokkos::deep_copy(cld_tau_lw_bnd, clouds_lw.tau); + + // Do subcolumn sampling to map bands -> gpoints based on cloud fraction and overlap assumption; + // This implements the Monte Carlo Independing Column Approximation by mapping only a single + // subcolumn (cloud state) to each gpoint. + auto nswgpts = k_dist_sw_k.get_ngpt(); + auto clouds_sw_gpt = get_subsampled_clouds(ncol, nlay, nswbands, nswgpts, clouds_sw, k_dist_sw_k, cldfrac, p_lay); + // Longwave + auto nlwgpts = k_dist_lw_k.get_ngpt(); + auto clouds_lw_gpt = get_subsampled_clouds(ncol, nlay, nlwbands, nlwgpts, clouds_lw, k_dist_lw_k, cldfrac, p_lay); + + // Copy cloud properties to outputs (is this needed, or can we just use pointers?) + // Alternatively, just compute and output a subcolumn cloud mask + Kokkos::parallel_for(MDRP::template get<3>({nswgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_sw_gpt(icol,ilay,igpt) = clouds_sw_gpt.tau(icol,ilay,igpt); + }); + Kokkos::parallel_for(MDRP::template get<3>({nlwgpts, nlay, ncol}), KOKKOS_LAMBDA (int igpt, int ilay, int icol) { + cld_tau_lw_gpt(icol,ilay,igpt) = clouds_lw_gpt.tau(icol,ilay,igpt); + }); + +#ifdef SCREAM_RRTMGP_DEBUG + // Perform checks on optics; these would be caught by RRTMGP_EXPENSIVE_CHECKS in the RRTMGP code, + // but we might want to provide additional debug info here. NOTE: we may actually want to move this + // up higher in the code, I think optical props should go up higher since optical props are kind of + // a parameterization of their own, and we might want to swap different choices. These checks go here + // only because we need to run them on computed optical props, so if the optical props themselves get + // computed up higher, then perform these checks higher as well + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); + check_range_k(clouds_sw.ssa, 0, 1, "rrtmgp_main:clouds_sw.ssa"); + check_range_k(clouds_sw.g , -1, 1, "rrtmgp_main:clouds_sw.g "); + check_range_k(clouds_sw.tau, 0, std::numeric_limits::max(), "rrtmgp_main:clouds_sw.tau"); +#endif + + // Do shortwave + rrtmgp_sw( + ncol, nlay, + k_dist_sw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + sfc_alb_dir, sfc_alb_dif, mu0, aerosol_sw, clouds_sw_gpt, + fluxes_sw, clnclrsky_fluxes_sw, clrsky_fluxes_sw, clnsky_fluxes_sw, + tsi_scaling, logger, + extra_clnclrsky_diag, extra_clnsky_diag + ); + + // Do longwave + rrtmgp_lw( + ncol, nlay, + k_dist_lw_k, p_lay, t_lay, p_lev, t_lev, gas_concs, + aerosol_lw, clouds_lw_gpt, + fluxes_lw, clnclrsky_fluxes_lw, clrsky_fluxes_lw, clnsky_fluxes_lw, + extra_clnclrsky_diag, extra_clnsky_diag + ); + +} + +/* + * Perform any clean-up tasks + */ +static void rrtmgp_finalize() +{ + initialized_k = false; + k_dist_sw_k.finalize(); + k_dist_lw_k.finalize(); + cloud_optics_sw_k.finalize(); //~CloudOptics(); + cloud_optics_lw_k.finalize(); //~CloudOptics(); + pool_t::finalize(); +} + +/* + * Shortwave driver (called by rrtmgp_main) + */ +static void rrtmgp_sw( + const int ncol, const int nlay, + gas_optics_t &k_dist, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + const creal2dk &sfc_alb_dir, const creal2dk &sfc_alb_dif, const real1dk &mu0, + optical_props2_t &aerosol, optical_props2_t &clouds, + fluxes_t &fluxes, fluxes_broadband_t &clnclrsky_fluxes, fluxes_broadband_t &clrsky_fluxes, fluxes_broadband_t &clnsky_fluxes, + const Real tsi_scaling, + const std::shared_ptr& logger, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) +{ + // Get problem sizes + int nbnd = k_dist.get_nband(); + int ngpt = k_dist.get_ngpt(); + int ngas = gas_concs.get_num_gases(); + + // Allocate temporaries from pool + const int size1 = nday; + const int size2 = nday*nlay; // 4 + const int size3 = nday*(nlay+1); // 5 + const int size4 = ncol*nlay; + const int size5 = nbnd*nday; //2 + const int size6 = nday*ngpt; + const int size7 = nday*(nlay+1)*nbnd; // 3 + const int size8 = ncol*nlay*(k_dist.get_ngas()+1); + + RealT* data = pool_t::template alloc_raw(size1 + size2*4 + size3*5 + size4 + size5*2 + size6 + size7*3 + size8), *dcurr = data; + + auto mu0_day = view_t (dcurr, nday); dcurr += size1; + + auto p_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto vmr_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_limited = view_t (dcurr, nday, nlay); dcurr += size2; + + auto p_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto t_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_up_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_dir_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + + auto vmr = view_t (dcurr, ncol, nlay); dcurr += size4; + + auto sfc_alb_dir_T = view_t (dcurr, nbnd, nday); dcurr += size5; + auto sfc_alb_dif_T = view_t (dcurr, nbnd, nday); dcurr += size5; + + auto toa_flux = view_t (dcurr, nday, ngpt); dcurr += size6; + + auto bnd_flux_up_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_dir_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + + auto col_gas = view_t(dcurr, ncol, nlay, k_dist.get_ngas()+1); dcurr += size8; + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &flux_dn_dir = fluxes.flux_dn_dir; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &bnd_flux_dn_dir = fluxes.bnd_flux_dn_dir; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clnclrsky_flux_dn_dir = clnclrsky_fluxes.flux_dn_dir; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clrsky_flux_dn_dir = clrsky_fluxes.flux_dn_dir; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + auto &clnsky_flux_dn_dir = clnsky_fluxes.flux_dn_dir; + + // Reset fluxes to zero + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up (icol,ilev) = 0; + flux_dn (icol,ilev) = 0; + flux_dn_dir(icol,ilev) = 0; + clnclrsky_flux_up (icol,ilev) = 0; + clnclrsky_flux_dn (icol,ilev) = 0; + clnclrsky_flux_dn_dir(icol,ilev) = 0; + clrsky_flux_up (icol,ilev) = 0; + clrsky_flux_dn (icol,ilev) = 0; + clrsky_flux_dn_dir(icol,ilev) = 0; + clnsky_flux_up (icol,ilev) = 0; + clnsky_flux_dn (icol,ilev) = 0; + clnsky_flux_dn_dir(icol,ilev) = 0; + }); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up (icol,ilev,ibnd) = 0; + bnd_flux_dn (icol,ilev,ibnd) = 0; + bnd_flux_dn_dir(icol,ilev,ibnd) = 0; + }); + + // Get daytime indices + auto dayIndices = view_t("dayIndices", ncol); + Kokkos::deep_copy(dayIndices, -1); + + int nday = 0; + // Serialized for now. + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, int& nday_inner) { + for (int icol = 0; icol < ncol; ++icol) { + if (mu0(icol) > 0) { + dayIndices(nday_inner++) = icol; + } + } + }, Kokkos::Sum(nday)); + + if (nday == 0) { + // No daytime columns in this chunk, skip the rest of this routine + return; + } + + // Subset mu0 + Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { + mu0_day(iday) = mu0(dayIndices(iday)); + }); + + // subset state variables + Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + p_lay_day(iday,ilay) = p_lay(dayIndices(iday),ilay); + t_lay_day(iday,ilay) = t_lay(dayIndices(iday),ilay); + }); + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + p_lev_day(iday,ilev) = p_lev(dayIndices(iday),ilev); + t_lev_day(iday,ilev) = t_lev(dayIndices(iday),ilev); + }); + + // Subset gases + auto gas_names = gas_concs.get_gas_names(); + gas_concs_t gas_concs_day; + gas_concs_day.init(gas_names, nday, nlay); + for (int igas = 0; igas < ngas; igas++) { + gas_concs.get_vmr(gas_names[igas], vmr); + Kokkos::parallel_for(MDRP::template get<2>({nlay,nday}), KOKKOS_LAMBDA(int ilay, int iday) { + vmr_day(iday,ilay) = vmr(dayIndices(iday),ilay); + }); + gas_concs_day.set_vmr(gas_names[igas], vmr_day); + } + + // Subset aerosol optics + optical_props2_t aerosol_day; + aerosol_day.init(k_dist.get_band_lims_wavenumber()); + aerosol_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,nday}), KOKKOS_LAMBDA(int ibnd, int ilay, int iday) { + aerosol_day.tau(iday,ilay,ibnd) = aerosol.tau(dayIndices(iday),ilay,ibnd); + aerosol_day.ssa(iday,ilay,ibnd) = aerosol.ssa(dayIndices(iday),ilay,ibnd); + aerosol_day.g (iday,ilay,ibnd) = aerosol.g (dayIndices(iday),ilay,ibnd); + }); + + // Subset cloud optics + // TODO: nbnd -> ngpt once we pass sub-sampled cloud state + optical_props2_t clouds_day; + clouds_day.init(k_dist.get_band_lims_wavenumber(), k_dist.get_band_lims_gpoint()); + clouds_day.alloc_2str(nday, nlay); + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,nday}), KOKKOS_LAMBDA(int igpt, int ilay, int iday) { + clouds_day.tau(iday,ilay,igpt) = clouds.tau(dayIndices(iday),ilay,igpt); + clouds_day.ssa(iday,ilay,igpt) = clouds.ssa(dayIndices(iday),ilay,igpt); + clouds_day.g (iday,ilay,igpt) = clouds.g (dayIndices(iday),ilay,igpt); + }); + + // RRTMGP assumes surface albedos have a screwy dimension ordering + // for some strange reason, so we need to transpose these; also do + // daytime subsetting in the same kernel + Kokkos::parallel_for(MDRP::template get<2>({nbnd,nday}), KOKKOS_LAMBDA(int ibnd, int icol) { + sfc_alb_dir_T(ibnd,icol) = sfc_alb_dir(dayIndices(icol),ibnd); + sfc_alb_dif_T(ibnd,icol) = sfc_alb_dif(dayIndices(icol),ibnd); + }); + + // Temporaries we need for daytime-only fluxes + fluxes_t fluxes_day; + fluxes_day.flux_up = flux_up_day; + fluxes_day.flux_dn = flux_dn_day; + fluxes_day.flux_dn_dir = flux_dn_dir_day; + fluxes_day.bnd_flux_up = bnd_flux_up_day; + fluxes_day.bnd_flux_dn = bnd_flux_dn_day; + fluxes_day.bnd_flux_dn_dir = bnd_flux_dn_dir_day; + + // Allocate space for optical properties + optical_props2_t optics; + optics.alloc_2str(nday, nlay, k_dist); + + optical_props2_t optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_2str(nday, nlay, k_dist); + } + + // Limit temperatures for gas optics look-up tables + limit_to_bounds_k(t_lay_day, k_dist_sw_k.get_temp_min(), k_dist_sw_k.get_temp_max(), t_lay_limited); + + // Do gas optics + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics, toa_flux); + if (extra_clnsky_diag) { + k_dist.gas_optics(nday, nlay, top_at_1, p_lay_day, p_lev_day, t_lay_limited, gas_concs_day, col_gas, optics_no_aerosols, toa_flux); + } + +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_sw:optics.tau"); + check_range_k(optics.ssa, 0, 1, "rrtmgp_sw:optics.ssa"); //, "optics.ssa"); + check_range_k(optics.g , -1, 1, "rrtmgp_sw:optics.g "); //, "optics.g" ); +#endif + + // Apply tsi_scaling + Kokkos::parallel_for(MDRP::template get<2>({ngpt,nday}), KOKKOS_LAMBDA(int igpt, int iday) { + toa_flux(iday,igpt) = tsi_scaling * toa_flux(iday,igpt); + }); + + if (extra_clnclrsky_diag) { + // Compute clear-clean-sky (just gas) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnclrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnclrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnclrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } + + // Combine gas and aerosol optics + aerosol_day.delta_scale(); + aerosol_day.increment(optics); + + // Compute clearsky (gas + aerosol) fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clrsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clrsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clrsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + + // Now merge in cloud optics and do allsky calculations + + // Combine gas and cloud optics + clouds_day.delta_scale(); + clouds_day.increment(optics); + // Compute fluxes on daytime columns + rte_sw(optics, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + flux_up (icol,ilev) = flux_up_day (iday,ilev); + flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,nday}), KOKKOS_LAMBDA(int ibnd, int ilev, int iday) { + const int icol = dayIndices(iday); + bnd_flux_up (icol,ilev,ibnd) = bnd_flux_up_day (iday,ilev,ibnd); + bnd_flux_dn (icol,ilev,ibnd) = bnd_flux_dn_day (iday,ilev,ibnd); + bnd_flux_dn_dir(icol,ilev,ibnd) = bnd_flux_dn_dir_day(iday,ilev,ibnd); + }); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds_day.increment(optics_no_aerosols); + // Compute cleansky (gas + clouds) fluxes on daytime columns + rte_sw(optics_no_aerosols, top_at_1, mu0_day, toa_flux, sfc_alb_dir_T, sfc_alb_dif_T, fluxes_day); + // Expand daytime fluxes to all columns + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,nday}), KOKKOS_LAMBDA(int ilev, int iday) { + const int icol = dayIndices(iday); + clnsky_flux_up (icol,ilev) = flux_up_day (iday,ilev); + clnsky_flux_dn (icol,ilev) = flux_dn_day (iday,ilev); + clnsky_flux_dn_dir(icol,ilev) = flux_dn_dir_day(iday,ilev); + }); + } + + pool_t::dealloc(data, dcurr - data); +} + +/* + * Longwave driver (called by rrtmgp_main) + */ +static void rrtmgp_lw( + const int ncol, const int nlay, + gas_optics_t &k_dist, + const creal2dk &p_lay, const creal2dk &t_lay, const creal2dk &p_lev, const creal2dk &t_lev, + gas_concs_t &gas_concs, + optical_props1_t &aerosol, optical_props1_t &clouds, + fluxes_t &fluxes, fluxes_broadband_t &clnclrsky_fluxes, fluxes_broadband_t &clrsky_fluxes, fluxes_broadband_t &clnsky_fluxes, + const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) +{ + // Problem size + int nbnd = k_dist.get_nband(); + int constexpr max_gauss_pts = 4; + + const int size1 = ncol; + const int size2 = nbnd*ncol; + const int size3 = max_gauss_pts*max_gauss_pts; + const int size4 = ncol*nlay; + const int size5 = ncol*(nlay+1); + const int size6 = ncol*nlay*(k_dist.get_ngas()+1); + + RealT* data = pool_t::template alloc_raw(size1 + size2 + size3*2 + size4 + size5 + size6), *dcurr = data; + + view_t t_sfc (dcurr, ncol); dcurr += size1; + view_t emis_sfc (dcurr, nbnd,ncol); dcurr += size2; + view_t gauss_Ds (dcurr, max_gauss_pts,max_gauss_pts); dcurr += size3; + view_t gauss_wts (dcurr, max_gauss_pts,max_gauss_pts); dcurr += size3; + view_t t_lay_limited(dcurr, ncol, nlay); dcurr += size4; + view_t t_lev_limited(dcurr, ncol, nlay+1); dcurr += size5; + view_t col_gas (dcurr, std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); dcurr += size6; + + // Associate local pointers for fluxes + auto &flux_up = fluxes.flux_up; + auto &flux_dn = fluxes.flux_dn; + auto &bnd_flux_up = fluxes.bnd_flux_up; + auto &bnd_flux_dn = fluxes.bnd_flux_dn; + auto &clnclrsky_flux_up = clnclrsky_fluxes.flux_up; + auto &clnclrsky_flux_dn = clnclrsky_fluxes.flux_dn; + auto &clrsky_flux_up = clrsky_fluxes.flux_up; + auto &clrsky_flux_dn = clrsky_fluxes.flux_dn; + auto &clnsky_flux_up = clnsky_fluxes.flux_up; + auto &clnsky_flux_dn = clnsky_fluxes.flux_dn; + + // Reset fluxes to zero + Kokkos::parallel_for( + MDRP::template get<2>({nlay + 1, ncol}), KOKKOS_LAMBDA(int ilev, int icol) { + flux_up(icol, ilev) = 0; + flux_dn(icol, ilev) = 0; + clnclrsky_flux_up(icol, ilev) = 0; + clnclrsky_flux_dn(icol, ilev) = 0; + clrsky_flux_up(icol, ilev) = 0; + clrsky_flux_dn(icol, ilev) = 0; + clnsky_flux_up(icol, ilev) = 0; + clnsky_flux_dn(icol, ilev) = 0; + }); + Kokkos::parallel_for( + MDRP::template get<3>({nbnd, nlay + 1, ncol}), + KOKKOS_LAMBDA(int ibnd, int ilev, int icol) { + bnd_flux_up(icol, ilev, ibnd) = 0; + bnd_flux_dn(icol, ilev, ibnd) = 0; + }); + + // Allocate space for optical properties + optical_props1_t optics; + optics.alloc_1scl(ncol, nlay, k_dist); + optical_props1_t optics_no_aerosols; + if (extra_clnsky_diag) { + // Allocate space for optical properties (no aerosols) + optics_no_aerosols.alloc_1scl(ncol, nlay, k_dist); + } + + // Boundary conditions + source_func_t lw_sources; + lw_sources.alloc(ncol, nlay, k_dist); + + bool top_at_1 = false; + Kokkos::parallel_reduce(1, KOKKOS_LAMBDA(int, bool& val) { + val |= p_lay(0, 0) < p_lay(0, nlay-1); + }, Kokkos::LOr(top_at_1)); + + // Surface temperature + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + t_sfc(icol) = t_lev(icol, conv::merge(nlay, 0, top_at_1)); + }); + Kokkos::deep_copy(emis_sfc , 0.98); + + // Get Gaussian quadrature weights + // TODO: move this crap out of userland! + // Weights and angle secants for first order (k=1) Gaussian quadrature. + // Values from Table 2, Clough et al, 1992, doi:10.1029/92JD01419 + // after Abramowitz & Stegun 1972, page 921 + RealT gauss_Ds_host_raw[max_gauss_pts][max_gauss_pts] = { + {1.66, 1.18350343, 1.09719858, 1.06056257}, + {0., 2.81649655, 1.69338507, 1.38282560}, + {0., 0., 4.70941630, 2.40148179}, + {0., 0., 0., 7.15513024} + }; + hview_t gauss_Ds_host (&gauss_Ds_host_raw[0][0], max_gauss_pts, max_gauss_pts); + + RealT gauss_wts_host_raw[max_gauss_pts][max_gauss_pts] = { + {0.5, 0.3180413817, 0.2009319137, 0.1355069134}, + {0., 0.1819586183, 0.2292411064, 0.2034645680}, + {0., 0., 0.0698269799, 0.1298475476}, + {0., 0., 0., 0.0311809710} + }; + + hview_t gauss_wts_host(&gauss_wts_host_raw[0][0],max_gauss_pts,max_gauss_pts); + + Kokkos::deep_copy(gauss_Ds, gauss_Ds_host); + Kokkos::deep_copy(gauss_wts, gauss_wts_host); + + // Limit temperatures for gas optics look-up tables + limit_to_bounds_k(t_lay, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lay_limited); + limit_to_bounds_k(t_lev, k_dist_lw_k.get_temp_min(), k_dist_lw_k.get_temp_max(), t_lev_limited); + + // Do gas optics + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics, lw_sources, view_t(), t_lev_limited); + if (extra_clnsky_diag) { + k_dist.gas_optics(ncol, nlay, top_at_1, p_lay, p_lev, t_lay_limited, t_sfc, gas_concs, col_gas, optics_no_aerosols, lw_sources, view_t(), t_lev_limited); + } + +#ifdef SCREAM_RRTMGP_DEBUG + // Check gas optics + check_range_k(optics.tau, 0, std::numeric_limits::max(), "rrtmgp_lw:optics.tau"); +#endif + + if (extra_clnclrsky_diag) { + // Compute clean-clear-sky fluxes before we add in aerosols and clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clnclrsky_fluxes); + } + + // Combine gas and aerosol optics + aerosol.increment(optics); + + // Compute clear-sky fluxes before we add in clouds + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, clrsky_fluxes); + + // Combine gas and cloud optics + clouds.increment(optics); + + // Compute allsky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics, top_at_1, lw_sources, emis_sfc, fluxes); + + if (extra_clnsky_diag) { + // First increment clouds in optics_no_aerosols + clouds.increment(optics_no_aerosols); + // Compute clean-sky fluxes + rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); + } + + pool_t::dealloc(data, dcurr - data); +} + +/* + * Return a subcolumn mask consistent with a specified overlap assumption + */ +static void get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds, int3dk& subcolumn_mask) +{ + // Subcolumn generators are a means for producing a variable x(i,j,k), where + // + // c(i,j,k) = 1 for x(i,j,k) > 1 - cldf(i,j) + // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) + // + // I am going to call this "cldx" to be just slightly less ambiguous + auto cldx = pool_t::template alloc(ncol, nlay, ngpt); + + // Apply overlap assumption to set cldx + if (overlap_option == 0) { // Dummy mask, always cloudy + Kokkos::deep_copy(cldx, 1); + } else { // Default case, maximum-random overlap + // Maximum-random overlap: + // Uses essentially the algorithm described in eq (14) in Raisanen et al. 2004, + // https://rmets.onlinelibrary.wiley.com/doi/epdf/10.1256/qj.03.99. Also the same + // algorithm used in RRTMG implementation of maximum-random overlap (see + // https://github.com/AER-RC/RRTMG_SW/blob/master/src/mcica_subcol_gen_sw.f90) + // + // First, fill cldx with random numbers. Need to use a unique seed for each column! + // auto seeds_host = Kokkos::create_mirror_view(seeds); + // Kokkos::deep_copy(seeds_host, seeds); + // for (int icol = 0; icol < ncol; ++icol) { + // Kokkos::Random_XorShift64_Pool<> random_pool(seeds_host(icol)); + // Kokkos::parallel_for(MDRP::template get<2>({ngpt, nlay}), KOKKOS_LAMBDA(int igpt, int ilay) { + // auto generator = random_pool.get_state(); + // cldx(icol,ilay,igpt) = generator.drand(0., 1.); + // random_pool.free_state(generator); + // }); + // } + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + conv::Random rand(seeds(icol)); + for (int igpt = 0; igpt < ngpt; igpt++) { + for (int ilay = 0; ilay < nlay; ilay++) { + cldx(icol,ilay,igpt) = rand.genFP(); + } + } + }); + + // Step down columns and apply algorithm from eq (14) + Kokkos::parallel_for(MDRP::template get<2>({ngpt,ncol}), KOKKOS_LAMBDA(int igpt, int icol) { + for (int ilay = 1; ilay < nlay; ilay++) { + // Check cldx in level above and see if it satisfies conditions to create a cloudy subcolumn + if (cldx(icol,ilay-1,igpt) > 1.0 - cldf(icol,ilay-1)) { + // Cloudy subcolumn above, use same random number here so that clouds in these two adjacent + // layers are maximimally overlapped + cldx(icol,ilay,igpt) = cldx(icol,ilay-1,igpt); + } else { + // Cloud-less above, use new random number so that clouds are distributed + // randomly in this layer. Need to scale new random number to range + // [0, 1.0 - cldf(ilay-1)] because we have artifically changed the distribution + // of random numbers in this layer with the above branch of the conditional, + // which would otherwise inflate cloud fraction in this layer. + cldx(icol,ilay,igpt) = cldx(icol,ilay ,igpt) * (1.0 - cldf(icol,ilay-1)); + } + } + }); + } + + // Use cldx array to create subcolumn mask + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + if (cldx(icol,ilay,igpt) > 1.0 - cldf(icol,ilay)) { + subcolumn_mask(icol,ilay,igpt) = 1; + } else { + subcolumn_mask(icol,ilay,igpt) = 0; + } + }); + + pool_t::dealloc(cldx); + + return subcolumn_mask; +} + +/* + * Compute cloud area from 3d subcol cloud property + */ +static void compute_cloud_area( + int ncol, int nlay, int ngpt, Real pmin, Real pmax, + const creal2dk& pmid, const real3dk& cld_tau_gpt, const real1dk& cld_area) +{ + // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy + // then 2d subcol mask is 1, otherwise it is 0 + auto subcol_mask = pool_t::template alloc(ncol, ngpt); + Kokkos::parallel_for(MDRP::template get<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but + // using play/pmid does not + if (cld_tau_gpt(icol,ilay,igpt) > 0 && pmid(icol,ilay) >= pmin && pmid(icol,ilay) < pmax) { + subcol_mask(icol,igpt) = 1; + } + }); + // Compute average over subcols to get cloud area + auto ngpt_inv = 1.0 / ngpt; + Kokkos::deep_copy(cld_area, 0); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // This loop needs to be serial because of the atomic reduction + for (int igpt = 0; igpt < ngpt; ++igpt) { + cld_area(icol) += subcol_mask(icol,igpt) * ngpt_inv; + } + }); + + pool_t::dealloc(subcol_mask); +} + +/* + * Return select cloud-top diagnostics following AeroCom recommendation + */ +static void compute_aerocom_cloudtop( + int ncol, int nlay, const creal2dk &tmid, const creal2dk &pmid, + const creal2dk &p_del, const real2dk &z_del, const creal2dk &qc, + const creal2dk &qi, const creal2dk &rel, const creal2dk &rei, + const real2dk &cldfrac_tot, const creal2dk &nc, + const real1dk &T_mid_at_cldtop, const real1dk &p_mid_at_cldtop, + const real1dk &cldfrac_ice_at_cldtop, const real1dk &cldfrac_liq_at_cldtop, + const real1dk &cldfrac_tot_at_cldtop, const real1dk &cdnc_at_cldtop, + const real1dk &eff_radius_qc_at_cldtop, const real1dk &eff_radius_qi_at_cldtop) +{ + /* The goal of this routine is to calculate properties at cloud top + * based on the AeroCom recommendation. See reference for routine + * get_subcolumn_mask above, where equation 14 is used for the + * maximum-random overlap assumption for subcolumn generation. We use + * equation 13, the column counterpart. + */ + // Set outputs to zero + Kokkos::deep_copy(T_mid_at_cldtop, 0.0); + Kokkos::deep_copy(p_mid_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_ice_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_liq_at_cldtop, 0.0); + Kokkos::deep_copy(cldfrac_tot_at_cldtop, 0.0); + Kokkos::deep_copy(cdnc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qc_at_cldtop, 0.0); + Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); + + // Initialize the 1D "clear fraction" as 1 (totally clear) + auto aerocom_clr = pool_t::template alloc(ncol); + Kokkos::deep_copy(aerocom_clr, 1.0); + + // Get gravity acceleration constant from constants + using physconst = scream::physics::Constants; + + // TODO: move tunable constant to namelist + constexpr RealT q_threshold = 0.0; // BAD_CONSTANT! + + // TODO: move tunable constant to namelist + constexpr RealT cldfrac_tot_threshold = 0.001; // BAD_CONSTANT! + + // Loop over all columns in parallel + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + // Loop over all layers in serial (due to accumulative + // product), starting at 2 (second highest) layer because the + // highest is assumed to hav no clouds + for(int ilay = 1; ilay < nlay; ++ilay) { + // Only do the calculation if certain conditions are met + if((qc(icol, ilay) + qi(icol, ilay)) > q_threshold && + (cldfrac_tot(icol, ilay) > cldfrac_tot_threshold)) { + /* PART I: Probabilistically determining cloud top */ + // Populate aerocom_tmp as the clear-sky fraction + // probability of this level, where aerocom_clr is that of + // the previous level + auto aerocom_tmp = + aerocom_clr(icol) * + (1.0 - ekat::impl::max(cldfrac_tot(icol, ilay - 1), + cldfrac_tot(icol, ilay))) / + (1.0 - ekat::impl::min(cldfrac_tot(icol, ilay - 1), + 1.0 - cldfrac_tot_threshold)); + // Temporary variable for probability "weights" + auto aerocom_wts = aerocom_clr(icol) - aerocom_tmp; + // Temporary variable for liquid "phase" + auto aerocom_phi = + qc(icol, ilay) / (qc(icol, ilay) + qi(icol, ilay)); + /* PART II: The inferred properties */ + /* In general, converting a 3D property X to a 2D cloud-top + * counterpart x follows: x(i) += X(i,k) * weights * Phase + * but X and Phase are not always needed */ + // T_mid_at_cldtop + T_mid_at_cldtop(icol) += tmid(icol, ilay) * aerocom_wts; + // p_mid_at_cldtop + p_mid_at_cldtop(icol) += pmid(icol, ilay) * aerocom_wts; + // cldfrac_ice_at_cldtop + cldfrac_ice_at_cldtop(icol) += + (1.0 - aerocom_phi) * aerocom_wts; + // cldfrac_liq_at_cldtop + cldfrac_liq_at_cldtop(icol) += aerocom_phi * aerocom_wts; + // cdnc_at_cldtop + /* We need to convert nc from 1/mass to 1/volume first, and + * from grid-mean to in-cloud, but after that, the + * calculation follows the general logic */ + auto cdnc = nc(icol, ilay) * p_del(icol, ilay) / + z_del(icol, ilay) / physconst::gravit / + cldfrac_tot(icol, ilay); + cdnc_at_cldtop(icol) += cdnc * aerocom_phi * aerocom_wts; + // eff_radius_qc_at_cldtop + eff_radius_qc_at_cldtop(icol) += + rel(icol, ilay) * aerocom_phi * aerocom_wts; + // eff_radius_qi_at_cldtop + eff_radius_qi_at_cldtop(icol) += + rei(icol, ilay) * (1.0 - aerocom_phi) * aerocom_wts; + // Reset aerocom_clr to aerocom_tmp to accumulate + aerocom_clr(icol) = aerocom_tmp; + } + } + // After the serial loop over levels, the cloudy fraction is + // defined as (1 - aerocom_clr). This is true because + // aerocom_clr is the result of accumulative probabilities + // (their products) + cldfrac_tot_at_cldtop(icol) = 1.0 - aerocom_clr(icol); + }); + + pool_t::dealloc(aerocom_clr); +} + +/* + * Provide a function to convert cloud (water and ice) mixing ratios to layer mass per unit area + * (what E3SM refers to as "in-cloud water paths", a terminology we shun here to avoid confusion + * with the standard practice of using "water path" to refer to the total column-integrated + * quantities). + */ template -void mixing_ratio_to_cloud_mass( +static void mixing_ratio_to_cloud_mass( View1 const& mixing_ratio, View2 const& cloud_fraction, View3 const& dp, @@ -315,7 +1211,7 @@ void mixing_ratio_to_cloud_mass( int ncol = mixing_ratio.extent(0); int nlay = mixing_ratio.extent(1); using physconst = scream::physics::Constants; - Kokkos::parallel_for(conv::get_mdrp<2>({nlay, ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay, ncol}), KOKKOS_LAMBDA(int ilay, int icol) { // Compute in-cloud mixing ratio (mixing ratio of the cloudy part of the layer) // NOTE: these thresholds (from E3SM) seem arbitrary, but included here for consistency // This limits in-cloud mixing ratio to 0.005 kg/kg. According to note in cloud_diagnostics @@ -329,7 +1225,6 @@ void mixing_ratio_to_cloud_mass( } }); } -#endif /* * Routine to limit a quantity to set bounds. Used to make sure @@ -337,33 +1232,227 @@ void mixing_ratio_to_cloud_mass( * property look-up tables, but could be used to limit other * fields as well. */ -#ifdef RRTMGP_ENABLE_YAKL -template -void limit_to_bounds(S const &arr_in, T const lower, T const upper, S &arr_out) { - yakl::c::parallel_for(arr_in.totElems(), YAKL_LAMBDA(int i) { - arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); +template::type* dummy = nullptr> +static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { + Kokkos::parallel_for(arr_out.size(), KOKKOS_LAMBDA(int i) { + arr_out(i) = std::min(std::max(arr_in(i), lower), upper); }); } -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -template -void limit_to_bounds_k(S const &arr_in, T const lower, T const upper, S &arr_out) { - Kokkos::parallel_for(arr_in.size(), KOKKOS_LAMBDA(int i) { - arr_out.data()[i] = std::min(std::max(arr_in.data()[i], lower), upper); + +template::type* dummy = nullptr> +static void limit_to_bounds_k(InT const &arr_in, T const lower, T const upper, OutT &arr_out) { + Kokkos::parallel_for(MDRP::template get<2>({arr_out.extent(0), arr_out.extent(1)}), KOKKOS_LAMBDA(int i, int j) { + arr_out(i, j) = std::min(std::max(arr_in(i, j), lower), upper); }); } -#endif -#ifdef RRTMGP_ENABLE_YAKL -int get_wavelength_index(OpticalProps &kdist, double wavelength); -int get_wavelength_index_sw(double wavelength); -int get_wavelength_index_lw(double wavelength); -#endif -#ifdef RRTMGP_ENABLE_KOKKOS -int get_wavelength_index(OpticalPropsK &kdist, double wavelength); -int get_wavelength_index_sw_k(double wavelength); -int get_wavelength_index_lw_k(double wavelength); -#endif + +static int get_wavelength_index(optical_props_t &kdist, RealT wavelength) +{ + // Get wavelength bounds for all wavelength bands + auto wavelength_bounds = kdist.get_band_lims_wavelength(); + + // Find the band index for the specified wavelength + // Note that bands are stored in wavenumber space, units of cm-1, so if we are passed wavelength + // in units of meters, we need a conversion factor of 10^2 + const int nbnds = kdist.get_nband(); + int band_index = -1; + Kokkos::parallel_reduce(nbnds, KOKKOS_LAMBDA(int ibnd, int& band_index_inner) { + if (wavelength_bounds(0,ibnd) < wavelength_bounds(1,ibnd)) { + if (wavelength_bounds(0,ibnd) <= wavelength * 1e2 && wavelength * 1e2 <= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } else { + if (wavelength_bounds(0,ibnd) >= wavelength * 1e2 && wavelength * 1e2 >= wavelength_bounds(1,ibnd)) { + band_index_inner = ibnd; + } + } + }, Kokkos::Max(band_index)); + return band_index; +} + +static inline int get_wavelength_index_sw_k(RealT wavelength) +{ + return get_wavelength_index(k_dist_sw_k, wavelength); +} + +static inline int get_wavelength_index_lw_k(RealT wavelength) +{ + return get_wavelength_index(k_dist_lw_k, wavelength); +} + +static optical_props2_t get_cloud_optics_sw( + const int ncol, const int nlay, + cloud_optics_t &cloud_optics, gas_optics_t &kdist, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei) { + + // Initialize optics + optical_props2_t clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_2str(ncol, nlay); + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = pool_t::template alloc(ncol, nlay); + auto rei_limited = pool_t::template alloc(ncol, nlay); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + pool_t::dealloc(rel_limited); + pool_t::dealloc(rei_limited); + + // Return optics + return clouds; +} + +static optical_props1_t get_cloud_optics_lw( + const int ncol, const int nlay, + cloud_optics_t &cloud_optics, gas_optics_t &kdist, + const real2dk &lwp, const real2dk &iwp, const creal2dk &rel, const creal2dk &rei) { + + // Initialize optics + optical_props1_t clouds; + clouds.init(kdist.get_band_lims_wavenumber()); + clouds.alloc_1scl(ncol, nlay); // this is dumb, why do we need to init and alloc separately?! + + // Needed for consistency with all-sky example problem? + cloud_optics.set_ice_roughness(2); + + // Limit effective radii to be within bounds of lookup table + auto rel_limited = pool_t::alloc(ncol, nlay); + auto rei_limited = pool_t::alloc(ncol, nlay); + limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); + limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); + + // Calculate cloud optics + cloud_optics.cloud_optics(ncol, nlay, lwp, iwp, rel_limited, rei_limited, clouds); + + pool_t::dealloc(rel_limited); + pool_t::dealloc(rei_limited); + + // Return optics + return clouds; +} + +static optical_props2_t get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + optical_props2_t &cloud_optics, gas_optics_t &kdist, const real2dk &cld, const creal2dk &p_lay) { + // Initialized subsampled optics + optical_props2_t subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_2str(ncol, nlay); + + // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto cldmask = pool_t::alloc(ncol, nlay, ngpt); + + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = pool_t::alloc(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask; note that get_subcolumn_mask exposes overlap assumption as an option, + // but the only currently supported options are 0 (trivial all-or-nothing cloud) or 1 (max-rand), + // so overlap has not been exposed as an option beyond this subcolumn. In the future, we should + // support generalized overlap as well, with parameters derived from DPSCREAM simulations with very + // high resolution. + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM + auto seeds = pool_t::alloc(ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); + }); + get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds, cldmask); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + subsampled_optics.ssa(icol,ilay,igpt) = cloud_optics.ssa(icol,ilay,ibnd); + subsampled_optics.g (icol,ilay,igpt) = cloud_optics.g (icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + subsampled_optics.ssa(icol,ilay,igpt) = 0; + subsampled_optics.g (icol,ilay,igpt) = 0; + } + }); + + pool_t::dealloc(cldmask); + pool_t::dealloc(cldfrac_rad); + pool_t::dealloc(seeds); + + return subsampled_optics; +} + +static optical_props1_t get_subsampled_clouds( + const int ncol, const int nlay, const int nbnd, const int ngpt, + optical_props1_t &cloud_optics, gas_optics_t &kdist, const real2dk &cld, const creal2dk &p_lay) { + + // Initialized subsampled optics + optical_props1_t subsampled_optics; + subsampled_optics.init(kdist.get_band_lims_wavenumber(), kdist.get_band_lims_gpoint(), "subsampled_optics"); + subsampled_optics.alloc_1scl(ncol, nlay); + + // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud + auto cldmask = pool_t::alloc(ncol, nlay, ngpt); + + // Check that we do not have clouds with no optical properties; this would get corrected + // when we assign optical props, but we want to use a "radiative cloud fraction" + // for the subcolumn sampling too because otherwise we can get vertically-contiguous cloud + // mask profiles with no actual cloud properties in between, which would just further overestimate + // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped + // even when separated by layers with no cloud properties, when in fact those layers should be + // randomly overlapped. + auto cldfrac_rad = pool_t::alloc(ncol, nlay); + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { + if (cloud_optics.tau(icol,ilay,ibnd) > 0) { + cldfrac_rad(icol,ilay) = cld(icol,ilay); + } + }); + // Get subcolumn cloud mask + int overlap = 1; + // Get unique seeds for each column that are reproducible across different MPI rank layouts; + // use decimal part of pressure for this, consistent with the implementation in EAM; use different + // seed values for longwave and shortwave + auto seeds = pool_t::alloc(ncol); + Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { + seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); + }); + get_subcolumn_mask(ncol, nlay, ngpt, cldfrac_rad, overlap, seeds, cldmask); + // Assign optical properties to subcolumns (note this implements MCICA) + auto gpoint_bands = kdist.get_gpoint_bands(); + Kokkos::parallel_for(MDRP::template get<3>({ngpt,nlay,ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { + auto ibnd = gpoint_bands(igpt); + if (cldmask(icol,ilay,igpt) == 1) { + subsampled_optics.tau(icol,ilay,igpt) = cloud_optics.tau(icol,ilay,ibnd); + } else { + subsampled_optics.tau(icol,ilay,igpt) = 0; + } + }); + + pool_t::dealloc(cldmask); + pool_t::dealloc(cldfrac_rad); + pool_t::dealloc(seeds); + + return subsampled_optics; +} + +}; // struct rrtmgp_interface +#endif // RRTMGP_ENABLE_KOKKOS } // namespace rrtmgp } // namespace scream diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index 65d1da9f1d68..0d3f18e7d841 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -320,6 +320,12 @@ int run_yakl(int argc, char** argv) { int run_kokkos(int argc, char** argv) { using namespace ekat::logger; using logger_t = Logger; + using interface_t = scream::rrtmgp::rrtmgp_interface<>; + using utils_t = rrtmgpTest::rrtmgp_test_utils<>; + using MDRP = utils_t::MDRP; + using real1dk = interface_t::view_t; + using real2dk = interface_t::view_t; + using real3dk = interface_t::view_t; ekat::Comm comm(MPI_COMM_WORLD); auto logger = std::make_shared("",LogLevel::info,comm); @@ -374,7 +380,7 @@ int run_kokkos(int argc, char** argv) { real2dk sw_flux_dir_ref; real2dk lw_flux_up_ref; real2dk lw_flux_dn_ref; - rrtmgpTest::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + utils_t::read_fluxes(inputfile, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); // Get dimension sizes int ncol = sw_flux_up_ref.extent(0); @@ -394,12 +400,12 @@ int run_kokkos(int argc, char** argv) { real2dk p_lev("p_lev", ncol, nlay+1); real2dk t_lev("t_lev", ncol, nlay+1); real2dk col_dry; - GasConcsK gas_concs; + GasConcsK gas_concs; read_atmos(inputfile, p_lay, t_lay, p_lev, t_lev, gas_concs, col_dry, ncol); // Initialize absorption coefficients logger->info("Initialize RRTMGP...\n"); - scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + interface_t::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); // Setup our dummy atmosphere based on the input data we read in logger->info("Setup dummy atmos...\n"); @@ -413,7 +419,7 @@ int run_kokkos(int argc, char** argv) { real2dk rel("rel", ncol, nlay); real2dk rei("rei", ncol, nlay); real2dk cld("cld", ncol, nlay); - rrtmgpTest::dummy_atmos( + utils_t::dummy_atmos( inputfile, ncol, p_lay, t_lay, sfc_alb_dir_vis, sfc_alb_dir_nir, sfc_alb_dif_vis, sfc_alb_dif_nir, @@ -426,8 +432,8 @@ int run_kokkos(int argc, char** argv) { // we would just have to setup the pointers to them in the // FluxesBroadband object logger->info("Setup fluxes...\n"); - const auto nswbands = scream::rrtmgp::k_dist_sw_k.get_nband(); - const auto nlwbands = scream::rrtmgp::k_dist_lw_k.get_nband(); + const auto nswbands = interface_t::k_dist_sw_k.get_nband(); + const auto nlwbands = interface_t::k_dist_lw_k.get_nband(); real2dk sw_flux_up ("sw_flux_up" , ncol, nlay+1); real2dk sw_flux_dn ("sw_flux_dn" , ncol, nlay+1); real2dk sw_flux_dir("sw_flux_dir", ncol, nlay+1); @@ -457,7 +463,7 @@ int run_kokkos(int argc, char** argv) { // Compute band-by-band surface_albedos. real2dk sfc_alb_dir("sfc_alb_dir", ncol, nswbands); real2dk sfc_alb_dif("sfc_alb_dif", ncol, nswbands); - rrtmgp::compute_band_by_band_surface_albedos( + interface_t::compute_band_by_band_surface_albedos( ncol, nswbands, sfc_alb_dir_vis, sfc_alb_dir_nir, sfc_alb_dif_vis, sfc_alb_dif_nir, @@ -468,19 +474,19 @@ int run_kokkos(int argc, char** argv) { auto aer_ssa_sw = real3dk("aer_ssa_sw", ncol, nlay, nswbands); auto aer_asm_sw = real3dk("aer_asm_sw", ncol, nlay, nswbands); auto aer_tau_lw = real3dk("aer_tau_lw", ncol, nlay, nlwbands); - Kokkos::parallel_for(conv::get_mdrp<3>({nswbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nswbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { aer_tau_sw(icol,ilay,ibnd) = 0; aer_ssa_sw(icol,ilay,ibnd) = 0; aer_asm_sw(icol,ilay,ibnd) = 0; }); - Kokkos::parallel_for(conv::get_mdrp<3>({nlwbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nlwbands,nlay,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { aer_tau_lw(icol,ilay,ibnd) = 0; }); // These are returned as outputs now from rrtmgp_main // TODO: provide as inputs consistent with how aerosol is treated? - const auto nswgpts = scream::rrtmgp::k_dist_sw_k.get_ngpt(); - const auto nlwgpts = scream::rrtmgp::k_dist_lw_k.get_ngpt(); + const auto nswgpts = interface_t::k_dist_sw_k.get_ngpt(); + const auto nlwgpts = interface_t::k_dist_lw_k.get_ngpt(); auto cld_tau_sw_bnd = real3dk("cld_tau_sw_bnd", ncol, nlay, nswbands); auto cld_tau_lw_bnd = real3dk("cld_tau_lw_bnd", ncol, nlay, nlwbands); auto cld_tau_sw = real3dk("cld_tau_sw", ncol, nlay, nswgpts); @@ -489,7 +495,7 @@ int run_kokkos(int argc, char** argv) { // Run RRTMGP code on dummy atmosphere logger->info("Run RRTMGP...\n"); const Real tsi_scaling = 1; - scream::rrtmgp::rrtmgp_main( + interface_t::rrtmgp_main( ncol, nlay, p_lay, t_lay, p_lev, t_lev, gas_concs, sfc_alb_dir, sfc_alb_dif, mu0, @@ -513,33 +519,33 @@ int run_kokkos(int argc, char** argv) { // Check values against baseline logger->info("Check values...\n"); - rrtmgpTest::read_fluxes( + utils_t::read_fluxes( baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); int nerr = 0; - if (!rrtmgpTest::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_up_ref , sw_flux_up , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_dn_ref , sw_flux_dn , 0.001)) nerr++; + if (!utils_t::all_close(sw_flux_dir_ref, sw_flux_dir, 0.001)) nerr++; + if (!utils_t::all_close(lw_flux_up_ref , lw_flux_up , 0.001)) nerr++; + if (!utils_t::all_close(lw_flux_dn_ref , lw_flux_dn , 0.001)) nerr++; // Because the aerosol optical properties are all set to zero, these fluxes must be equal - if (!rrtmgpTest::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; - if (!rrtmgpTest::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_up , sw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_up , sw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_dn , sw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_dn , sw_clnclrsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_flux_dir , sw_clnsky_flux_dir , 0.0000000001)) nerr++; + if (!utils_t::all_close(sw_clrsky_flux_dir , sw_clnclrsky_flux_dir , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_flux_up , lw_clnsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_clrsky_flux_up , lw_clnclrsky_flux_up , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_flux_dn , lw_clnsky_flux_dn , 0.0000000001)) nerr++; + if (!utils_t::all_close(lw_clrsky_flux_dn , lw_clnclrsky_flux_dn , 0.0000000001)) nerr++; logger->info("Cleaning up...\n"); // Clean up or else YAKL will throw errors - scream::rrtmgp::rrtmgp_finalize(); + interface_t::rrtmgp_finalize(); scream::finalize_kls(); return nerr != 0 ? 1 : 0; diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index ddef61808fe0..99a582445616 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -849,6 +849,15 @@ TEST_CASE("rrtmgp_aerocom_cloudtop") { #endif #ifdef RRTMGP_ENABLE_KOKKOS +using interface_t = scream::rrtmgp::rrtmgp_interface<>; +using real1dk = interface_t::view_t; +using real2dk = interface_t::view_t; +using real3dk = interface_t::view_t; +using int1dk = interface_t::view_t; +using int2dk = interface_t::view_t; +using int3dk = interface_t::view_t; +using MDRP = interface_t::MDRP; + TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos scream::init_kls(); @@ -921,7 +930,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 1.0; }); auto cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with no cloud @@ -931,7 +940,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.0; }); cloud_mass_ref = 0.0; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with empty clouds (cloud fraction but with no associated mixing ratio) @@ -943,7 +952,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.1; }); cloud_mass_ref = 0.0; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Test with cell half filled with cloud @@ -953,7 +962,7 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { cloud_fraction(0,0) = 0.5; }); cloud_mass_ref = chc(mixing_ratio)(0,0) / chc(cloud_fraction)(0,0) * chc(dp)(0,0) / physconst::gravit; - scream::rrtmgp::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); + interface_t::mixing_ratio_to_cloud_mass(mixing_ratio, cloud_fraction, dp, cloud_mass); REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Clean up @@ -977,14 +986,14 @@ TEST_CASE("rrtmgp_test_limit_to_bounds_k") { }); // Limit to bounds that contain the data; should be no change in values - scream::rrtmgp::limit_to_bounds(arr, 0.0, 5.0, arr_limited); + interface_t::limit_to_bounds_k(arr, 0.0, 5.0, arr_limited); REQUIRE(chc(arr)(0,0) == chc(arr_limited)(0,0)); REQUIRE(chc(arr)(0,1) == chc(arr_limited)(0,1)); REQUIRE(chc(arr)(1,0) == chc(arr_limited)(1,0)); REQUIRE(chc(arr)(1,1) == chc(arr_limited)(1,1)); // Limit to bounds that do not completely contain the data; should be a change in values! - scream::rrtmgp::limit_to_bounds(arr, 1.5, 3.5, arr_limited); + interface_t::limit_to_bounds_k(arr, 1.5, 3.5, arr_limited); REQUIRE(chc(arr_limited)(0,0) == 1.5); REQUIRE(chc(arr_limited)(0,1) == 2.0); REQUIRE(chc(arr_limited)(1,0) == 3.0); @@ -1074,11 +1083,11 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Need to initialize RRTMGP with dummy gases logger->info("Init gases...\n"); - GasConcsK gas_concs; + GasConcsK gas_concs; string1dv gas_names = {"h2o", "co2", "o3", "n2o", "co", "ch4", "o2", "n2"}; gas_concs.init(gas_names,ncol,nlay); logger->info("Init RRTMGP...\n"); - scream::rrtmgp::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); + interface_t::rrtmgp_initialize(gas_concs, coefficients_file_sw, coefficients_file_lw, cloud_optics_file_sw, cloud_optics_file_lw, logger); // Create simple test cases; We expect, given the input data, that band 10 // will straddle the NIR and VIS, bands 1-9 will be purely NIR, and bands 11-14 @@ -1092,7 +1101,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { auto sw_bnd_flux_dir = real3dk("sw_bnd_flux_dir", ncol, nlay+1, nbnd); auto sw_bnd_flux_dif = real3dk("sw_bnd_flux_dif", ncol, nlay+1, nbnd); logger->info("Populate band-resolved 3d fluxes for test case with only transition band flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; @@ -1106,7 +1115,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1124,7 +1133,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in NIR bands logger->info("Populate band-resolved 3d fluxes for test case with only NIR flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1; sw_bnd_flux_dif(icol,ilay,ibnd) = 1; @@ -1138,7 +1147,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1155,7 +1164,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in VIS bands logger->info("Populate band-resolved 3d fluxes for test case with only VIS/UV flux...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 0; sw_bnd_flux_dif(icol,ilay,ibnd) = 0; @@ -1169,7 +1178,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1186,7 +1195,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // --------------------------------- // Test case, only flux in all bands logger->info("Populate band-resolved 3d fluxes for test with non-zero flux in all bands...\n"); - Kokkos::parallel_for(conv::get_mdrp<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay+1,ncol}), KOKKOS_LAMBDA(int ibnd, int ilay, int icol) { if (ibnd < 9) { sw_bnd_flux_dir(icol,ilay,ibnd) = 1.0; sw_bnd_flux_dif(icol,ilay,ibnd) = 2.0; @@ -1200,7 +1209,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { }); // Compute surface fluxes logger->info("Compute broadband surface fluxes...\n"); - scream::rrtmgp::compute_broadband_surface_fluxes( + interface_t::compute_broadband_surface_fluxes( ncol, kbot, nbnd, sw_bnd_flux_dir, sw_bnd_flux_dif, sfc_flux_dir_vis, sfc_flux_dir_nir, @@ -1216,7 +1225,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Finalize YAKL logger->info("Free memory...\n"); - scream::rrtmgp::rrtmgp_finalize(); + interface_t::rrtmgp_finalize(); gas_concs.reset(); scream::finalize_kls(); } @@ -1282,16 +1291,16 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { for (int igpt = 0; igpt < ngpt; ++igpt) { real cldmask_real = cldmask(icol,ilay,igpt); cldfrac_from_mask(icol,ilay) += cldmask_real; } }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { cldfrac_from_mask(icol,ilay) = cldfrac_from_mask(icol,ilay) / ngpt; }); // For cldfrac 1 we should get 1, for cldfrac 0 we should get 0, but in between we cannot be sure @@ -1316,7 +1325,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = scream::rrtmgp::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); auto cldmask_h = chc(cldmask); for (int igpt = 0; igpt < ngpt; igpt++) { if (cldmask_h(0,0,igpt) == 1) { @@ -1359,7 +1368,7 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 0; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 0.0); // Case: @@ -1376,7 +1385,7 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 1; cldtau(0,1,2) = 1; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0); // Case: @@ -1393,11 +1402,11 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 1.0; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 150, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, 150, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 110, 250, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 110, 250, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 1.0 / 3.0); // Case: @@ -1414,11 +1423,11 @@ TEST_CASE("rrtmgp_cloud_area_k") { cldtau(0,1,1) = 0; cldtau(0,1,2) = 1; }); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, std::numeric_limits::max(), pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 0, 100, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 0, 100, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 0.0); - scream::rrtmgp::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); + interface_t::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); scream::finalize_kls(); } @@ -1463,7 +1472,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { Kokkos::deep_copy(rel, 10.0); Kokkos::deep_copy(rei, 10.0); // Call the function - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1481,7 +1490,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { // Case 2: if all clouds, everything goes to 1 * its value Kokkos::deep_copy(cldfrac_tot, 1.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1504,7 +1513,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 3) = 0.3; cldfrac_tot(0, 4) = 0.2; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1520,7 +1529,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 5) = 0.4; cldfrac_tot(0, 6) = 0.2; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1534,7 +1543,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { cldfrac_tot(0, 4) = 0.0; cldfrac_tot(0, 5) = 0.1; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1550,7 +1559,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { }); Kokkos::deep_copy(qc, 1.0); Kokkos::deep_copy(qi, 0.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1568,7 +1577,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { }); Kokkos::deep_copy(qc, 0.0); Kokkos::deep_copy(qi, 1.0); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, @@ -1604,7 +1613,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { qc(0, 6) = 50; qc(0, 7) = 10; }); - scream::rrtmgp::compute_aerocom_cloudtop( + interface_t::compute_aerocom_cloudtop( ncol, nlay, tmid, pmid, p_del, z_del, qc, qi, rel, rei, cldfrac_tot, nc, tmid_at_cldtop, pmid_at_cldtop, cldfrac_ice_at_cldtop, cldfrac_liq_at_cldtop, cldfrac_tot_at_cldtop, cdnc_at_cldtop, diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index 891626b9d3df..4796b30abefd 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -104,14 +104,14 @@ if (HIP_BUILD) endif() set(SHOC_LIBS "shoc") -if (SCREAM_SMALL_KERNELS) +if (SCREAM_SHOC_SMALL_KERNELS) add_library(shoc ${SHOC_SRCS} ${SHOC_SK_SRCS}) else() add_library(shoc ${SHOC_SRCS}) if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(shoc_sk ${SHOC_SRCS} ${SHOC_SK_SRCS}) - # Always build shoc_sk with SCREAM_SMALL_KERNELS on - target_compile_definitions(shoc_sk PUBLIC "SCREAM_SMALL_KERNELS") + # Always build shoc_sk with SCREAM_SHOC_SMALL_KERNELS on + target_compile_definitions(shoc_sk PUBLIC "SCREAM_SHOC_SMALL_KERNELS") list(APPEND SHOC_LIBS "shoc_sk") endif() endif() diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index 15d43c82f7c6..99e0d83ebf92 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -88,6 +88,27 @@ void SHOCMacrophysics::set_grids(const std::shared_ptr grids add_field("eddy_diff_heat", scalar3d_mid, m2/s, grid_name, ps); add_field("w_variance", scalar3d_mid, m2/s2, grid_name, ps); add_field("cldfrac_liq_prev", scalar3d_mid, nondim, grid_name, ps); + add_field("ustar", scalar2d, m/s, grid_name, ps); + add_field("obklen", scalar2d, m, grid_name, ps); + + // Extra SHOC output diagnostics + if (m_params.get("extra_shoc_diags", false)) { + + // Diagnostic output - mid point grid + add_field("brunt", scalar3d_mid, pow(s,-1), grid_name, ps); + add_field("shoc_mix", scalar3d_mid, m, grid_name, ps); + add_field("isotropy", scalar3d_mid, s, grid_name, ps); + + // Diagnostic output - interface grid + add_field("wthl_sec", scalar3d_int, K*(m/s), grid_name, ps); + add_field("thl_sec", scalar3d_int, pow(K,2), grid_name, ps); + add_field("wqw_sec", scalar3d_int, (kg/kg)*(m/s), grid_name, ps); + add_field("qw_sec", scalar3d_int, pow(kg/kg,2), grid_name, ps); + add_field("uw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); + add_field("vw_sec", scalar3d_int, pow(m/s,2), grid_name, ps); + add_field("w3", scalar3d_int, pow(m/s,3), grid_name, ps); + + } // Extra SHOC output diagnostics // Tracer group add_group("tracers", grid_name, ps, Bundling::Required); @@ -154,10 +175,10 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) using scalar_view_t = decltype(m_buffer.wpthlp_sfc); scalar_view_t* _1d_scalar_view_ptrs[Buffer::num_1d_scalar_ncol] = {&m_buffer.wpthlp_sfc, &m_buffer.wprtp_sfc, &m_buffer.upwp_sfc, &m_buffer.vpwp_sfc -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , &m_buffer.se_b, &m_buffer.ke_b, &m_buffer.wv_b, &m_buffer.wl_b , &m_buffer.se_a, &m_buffer.ke_a, &m_buffer.wv_a, &m_buffer.wl_a - , &m_buffer.ustar, &m_buffer.kbfs, &m_buffer.obklen, &m_buffer.ustar2, &m_buffer.wstar + , &m_buffer.kbfs, &m_buffer.ustar2, &m_buffer.wstar #endif }; for (int i = 0; i < Buffer::num_1d_scalar_ncol; ++i) { @@ -180,7 +201,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) &m_buffer.z_mid, &m_buffer.rrho, &m_buffer.thv, &m_buffer.dz, &m_buffer.zt_grid, &m_buffer.wm_zt, &m_buffer.inv_exner, &m_buffer.thlm, &m_buffer.qw, &m_buffer.dse, &m_buffer.tke_copy, &m_buffer.qc_copy, &m_buffer.shoc_ql2, &m_buffer.shoc_mix, &m_buffer.isotropy, &m_buffer.w_sec, &m_buffer.wqls_sec, &m_buffer.brunt -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , &m_buffer.rho_zt, &m_buffer.shoc_qv, &m_buffer.tabs, &m_buffer.dz_zt #endif }; @@ -189,7 +210,7 @@ void SHOCMacrophysics::init_buffers(const ATMBufferManager &buffer_manager) &m_buffer.z_int, &m_buffer.rrho_i, &m_buffer.zi_grid, &m_buffer.thl_sec, &m_buffer.qw_sec, &m_buffer.qwthl_sec, &m_buffer.wthl_sec, &m_buffer.wqw_sec, &m_buffer.wtke_sec, &m_buffer.uw_sec, &m_buffer.vw_sec, &m_buffer.w3 -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , &m_buffer.dz_zi #endif }; @@ -332,6 +353,8 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) output.pblh = get_field_out("pbl_height").get_view(); output.shoc_ql2 = shoc_ql2; output.tkh = get_field_out("eddy_diff_heat").get_view(); + output.ustar = get_field_out("ustar").get_view(); + output.obklen = get_field_out("obklen").get_view(); // Ouput (diagnostic) history_output.shoc_mix = m_buffer.shoc_mix; @@ -349,7 +372,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) history_output.wqls_sec = m_buffer.wqls_sec; history_output.brunt = m_buffer.brunt; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS temporaries.se_b = m_buffer.se_b; temporaries.ke_b = m_buffer.ke_b; temporaries.wv_b = m_buffer.wv_b; @@ -358,9 +381,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) temporaries.ke_a = m_buffer.ke_a; temporaries.wv_a = m_buffer.wv_a; temporaries.wl_a = m_buffer.wl_a; - temporaries.ustar = m_buffer.ustar; temporaries.kbfs = m_buffer.kbfs; - temporaries.obklen = m_buffer.obklen; temporaries.ustar2 = m_buffer.ustar2; temporaries.wstar = m_buffer.wstar; @@ -483,7 +504,7 @@ void SHOCMacrophysics::run_impl (const double dt) // Run shoc main SHF::shoc_main(m_num_cols, m_num_levs, m_num_levs+1, m_npbl, m_nadv, m_num_tracers, dt, workspace_mgr,runtime_options,input,input_output,output,history_output -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , temporaries #endif ); @@ -493,6 +514,41 @@ void SHOCMacrophysics::run_impl (const double dt) default_policy, shoc_postprocess); Kokkos::fence(); + + // Extra SHOC output diagnostics + if (m_params.get("extra_shoc_diags", false)) { + + const auto& shoc_mix = get_field_out("shoc_mix").get_view(); + Kokkos::deep_copy(shoc_mix,history_output.shoc_mix); + + const auto& brunt = get_field_out("brunt").get_view(); + Kokkos::deep_copy(brunt,history_output.brunt); + + const auto& w3 = get_field_out("w3").get_view(); + Kokkos::deep_copy(w3,history_output.w3); + + const auto& isotropy = get_field_out("isotropy").get_view(); + Kokkos::deep_copy(isotropy,history_output.isotropy); + + const auto& wthl_sec = get_field_out("wthl_sec").get_view(); + Kokkos::deep_copy(wthl_sec,history_output.wthl_sec); + + const auto& wqw_sec = get_field_out("wqw_sec").get_view(); + Kokkos::deep_copy(wqw_sec,history_output.wqw_sec); + + const auto& uw_sec = get_field_out("uw_sec").get_view(); + Kokkos::deep_copy(uw_sec,history_output.uw_sec); + + const auto& vw_sec = get_field_out("vw_sec").get_view(); + Kokkos::deep_copy(vw_sec,history_output.vw_sec); + + const auto& qw_sec = get_field_out("qw_sec").get_view(); + Kokkos::deep_copy(qw_sec,history_output.qw_sec); + + const auto& thl_sec = get_field_out("thl_sec").get_view(); + Kokkos::deep_copy(thl_sec,history_output.thl_sec); + + } // Extra SHOC output diagnostics } // ========================================================================================= void SHOCMacrophysics::finalize_impl() diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp index 6dfadb1eee7b..4d76efd4558b 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.hpp @@ -88,13 +88,10 @@ class SHOCMacrophysics : public scream::AtmosphereProcess cldfrac_liq_prev(i,k)=cldfrac_liq(i,k); - const auto range = ekat::range(k*Spack::n); - const Smask in_nlev_range = (range < nlev); - - // Inverse of Exner. Assert that exner != 0 when in range before computing. + // Inverse of Exner. In non-rel builds, assert that exner != 0 when in range before computing. const Spack exner = PF::exner_function(p_mid(i,k)); const Smask nonzero = (exner != 0); - EKAT_KERNEL_ASSERT((nonzero || !in_nlev_range).all()); + EKAT_KERNEL_ASSERT((nonzero || !(ekat::range(k*Spack::n) < nlev)).all()); inv_exner(i,k).set(nonzero, 1/exner); tke(i,k) = ekat::max(mintke, tke(i,k)); @@ -385,13 +382,13 @@ class SHOCMacrophysics : public scream::AtmosphereProcess // Structure for storing local variables initialized using the ATMBufferManager struct Buffer { -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS static constexpr int num_1d_scalar_ncol = 4; #else - static constexpr int num_1d_scalar_ncol = 17; + static constexpr int num_1d_scalar_ncol = 15; #endif static constexpr int num_1d_scalar_nlev = 1; -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS static constexpr int num_2d_vector_mid = 18; static constexpr int num_2d_vector_int = 12; #else @@ -404,7 +401,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess uview_1d wprtp_sfc; uview_1d upwp_sfc; uview_1d vpwp_sfc; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS uview_1d se_b; uview_1d ke_b; uview_1d wv_b; @@ -413,9 +410,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess uview_1d ke_a; uview_1d wv_a; uview_1d wl_a; - uview_1d ustar; uview_1d kbfs; - uview_1d obklen; uview_1d ustar2; uview_1d wstar; #endif @@ -453,7 +448,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess uview_2d w3; uview_2d wqls_sec; uview_2d brunt; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS uview_2d rho_zt; uview_2d shoc_qv; uview_2d tabs; @@ -511,7 +506,7 @@ class SHOCMacrophysics : public scream::AtmosphereProcess SHF::SHOCOutput output; SHF::SHOCHistoryOutput history_output; SHF::SHOCRuntime runtime_options; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS SHF::SHOCTemporaries temporaries; #endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp new file mode 100644 index 000000000000..7a0633fe7f34 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp @@ -0,0 +1,41 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_BUOYANCY_FLUX_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_BUOYANCY_FLUX_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_buoyancy_flux. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_compute_buoyancy_flux( + const Spack& wthlsec, + const Spack& wqwsec, + const Spack& pval, + const Spack& wqls, + Spack& wthv_sec) +{ + const Scalar basepres = C::P0; + const Scalar rair = C::Rair; + const Scalar rv = C::RV; + const Scalar cp = C::CP; + const Scalar lcond = C::LatVap; + const Scalar basetemp = C::basetemp; + const Scalar epsterm = rair/rv; + + wthv_sec = wthlsec + ((1 - epsterm)/epsterm)*basetemp*wqwsec + + ((lcond/cp)*ekat::pow(basepres/pval, (rair/cp)) + - (1/epsterm)*basetemp)*wqls; +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp new file mode 100644 index 000000000000..dac01954db31 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp @@ -0,0 +1,39 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_CLOUD_LIQUID_VARIANCE_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_CLOUD_LIQUID_VARIANCE_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_cloud_liquid_variance. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_compute_cloud_liquid_variance( + const Spack& a, + const Spack& s1, + const Spack& ql1, + const Spack& C1, + const Spack& std_s1, + const Spack& s2, + const Spack& ql2, + const Spack& C2, + const Spack& std_s2, + const Spack& shoc_ql, + Spack& shoc_ql2) +{ + shoc_ql2 = ekat::max(0, a*(s1*ql1 + C1*ekat::square(std_s1)) + + (1 - a)*(s2*ql2 + C2*ekat::square(std_s2)) + - ekat::square(shoc_ql)); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp new file mode 100644 index 000000000000..ceb64c533a37 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp @@ -0,0 +1,33 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_LIQUID_WATER_FLUX_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_LIQUID_WATER_FLUX_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_liquid_water_flux. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_compute_liquid_water_flux( + const Spack& a, + const Spack& w1_1, + const Spack& w_first, + const Spack& ql1, + const Spack& w1_2, + const Spack& ql2, + Spack& wqls) +{ + wqls = a*((w1_1 - w_first)*ql1) + (1 - a)*((w1_2 - w_first)*ql2); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp new file mode 100644 index 000000000000..867ffe3fed3a --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_qs_impl.hpp @@ -0,0 +1,55 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_QS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_QS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU +#include "physics_functions.hpp" + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_qs. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_compute_qs( + const Spack& Tl1_1, + const Spack& Tl1_2, + const Spack& pval, + const Smask& active_entries, + Spack& qs1, + Spack& beta1, + Spack& qs2, + Spack& beta2) +{ + const Scalar rair = C::Rair; + const Scalar rv = C::RV; + const Scalar cp = C::CP; + const Scalar lcond = C::LatVap; + + // Compute MurphyKoop_svp + const int liquid = 0; + const Spack esval1_1 = scream::physics::Functions::MurphyKoop_svp(Tl1_1,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_1)"); + const Spack esval1_2 = scream::physics::Functions::MurphyKoop_svp(Tl1_2,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_2)"); + const Spack lstarn(lcond); + + qs1 = sp(0.622)*esval1_1/ekat::max(esval1_1, pval - esval1_1); + beta1 = (rair/rv)*(lstarn/(rair*Tl1_1))*(lstarn/(cp*Tl1_1)); + + // Only compute qs2 and beta2 if the two plumes are not equal + const Smask condition = (Tl1_1 != Tl1_2); + qs2 = qs1; + beta2 = beta1; + + qs2.set(condition, sp(0.622)*esval1_2/ekat::max(esval1_2, pval - esval1_2)); + beta2.set(condition, (rair/rv)*(lstarn/(rair*Tl1_2))*(lstarn/(cp*Tl1_2))); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp new file mode 100644 index 000000000000..5794e21456f7 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_s_impl.hpp @@ -0,0 +1,69 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_S_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_S_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_s. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_compute_s( + const Spack& qw1, + const Spack& qs, + const Spack& beta, + const Spack& pval, + const Spack& thl2, + const Spack& qw2, + const Spack& sqrtthl2, + const Spack& sqrtqw2, + const Spack& r_qwthl, + Spack& s, + Spack& std_s, + Spack& qn, + Spack& C) +{ + const Scalar rair = C::Rair; + const Scalar basepres = C::P0; + const Scalar cp = C::CP; + const Scalar lcond = C::LatVap; + const Scalar pi = C::Pi; + + const Scalar sqrt2(std::sqrt(Scalar(2.0))), sqrt2pi(std::sqrt(2*pi)); + + const Spack cthl=((1 + beta*qw1)/ekat::square(1 + beta*qs))*(cp/lcond)* + beta*qs*ekat::pow(pval/basepres, (rair/cp)); + const Spack cqt = 1/(1 + beta*qs); + + std_s = ekat::sqrt(ekat::max(0, + ekat::square(cthl)*thl2 + + ekat::square(cqt)*qw2 - 2*cthl*sqrtthl2*cqt*sqrtqw2*r_qwthl)); + const auto std_s_not_small = std_s > std::sqrt(Kokkos::Experimental::norm_min_v) * 100; + s = qw1-qs*((1 + beta*qw1)/(1 + beta*qs)); + if (std_s_not_small.any()) { + C.set(std_s_not_small, sp(0.5)*(1 + ekat::erf(s/(sqrt2*std_s)))); + } + C.set(!std_s_not_small && s > 0, 1); + const auto std_s_C_not_small = std_s_not_small && C != 0; + if (std_s_C_not_small.any()) { + qn.set(std_s_C_not_small, s*C+(std_s/sqrt2pi)*ekat::exp(-sp(0.5)*ekat::square(s/std_s))); + } + qn.set(!std_s_not_small && s > 0, s); + + // Checking to prevent empty clouds + const auto qn_le_zero = qn <= 0; + C.set(qn_le_zero,0); + qn.set(qn_le_zero,0); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp new file mode 100644 index 000000000000..837fe90e2991 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_sgs_liquid_impl.hpp @@ -0,0 +1,30 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_SGS_LIQUID_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_SGS_LIQUID_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_sgs_liquid. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_compute_sgs_liquid( + const Spack& a, + const Spack& ql1, + const Spack& ql2, + Spack& shoc_ql) +{ + shoc_ql = ekat::max(0, a*ql1 + (1 - a)*ql2); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp new file mode 100644 index 000000000000..e60a744ae93f --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_compute_temperature_impl.hpp @@ -0,0 +1,32 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_COMPUTE_TEMPERATURE_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_COMPUTE_TEMPERATURE_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_compute_temperature. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_compute_temperature( + const Spack& thl1, + const Spack& pval, + Spack& Tl1) +{ + constexpr Scalar basepres = C::P0; + constexpr Scalar rair = C::Rair; + constexpr Scalar cp = C::CP; + Tl1 = thl1/(ekat::pow(basepres/pval,(rair/cp))); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp index a407e83c2ee7..cd004adbd2d6 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_impl.hpp @@ -9,6 +9,20 @@ namespace scream { namespace shoc { +/* +Add some functions to avoid cuda compilation warnings +TODO: move this to ekat or something +*/ +#ifdef KOKKOS_ENABLE_CUDA +KOKKOS_INLINE_FUNCTION constexpr Real safe_min() { + return Kokkos::Experimental::norm_min_v; +} +#else +KOKKOS_INLINE_FUNCTION constexpr Real safe_min() { + return std::numeric_limits::min(); +} +#endif + /* * Implementation of shoc shoc_assumed_pdf. Clients should NOT * #include this file, but include shoc_functions.hpp instead. @@ -63,15 +77,6 @@ void Functions::shoc_assumed_pdf( const Scalar w_tol_sqd = 4e-4; const Scalar w_thresh = 0; const Scalar largeneg = SC::largeneg; - const Scalar rair = C::Rair; - const Scalar rv = C::RV; - const bool dothetal_skew = SC::dothetal_skew; - const Scalar basepres = C::P0; - const Scalar cp = C::CP; - const Scalar lcond = C::LatVap; - const Scalar pi = C::Pi; - const Scalar basetemp = C::basetemp; - const Scalar epsterm = rair/rv; const Scalar Tl_min = 100; // Interpolate many variables from interface grid to thermo grid @@ -111,121 +116,35 @@ void Functions::shoc_assumed_pdf( const auto sqrtqt = ekat::max(rt_tol,ekat::sqrt(qwsec)); // Find parameters for vertical velocity - Spack Skew_w(0), w1_1(w_first), w1_2(w_first), w2_1(0), w2_2(0), a(0.5); - { - const Smask condition = w_sec(k) > w_tol_sqd; - - const Scalar tmp_val(0.4); - const Scalar one_m_tmp_val(1 - tmp_val); - const Scalar sqrtw2t(std::sqrt(1-tmp_val)); - - Skew_w.set(condition, w3var/ekat::sqrt(ekat::cube(w_sec(k)))); - a.set(condition, - ekat::max(sp(0.01), - ekat::min(sp(0.99), - sp(0.5)*(1 - Skew_w*ekat::sqrt(1/(4*(one_m_tmp_val*one_m_tmp_val*one_m_tmp_val) - + ekat::square(Skew_w))))))); - - w1_1.set(condition, ekat::sqrt((1 - a)/a)*sqrtw2t); - w1_2.set(condition, -1*ekat::sqrt(a/(1 - a))*sqrtw2t); - w2_1.set(condition, tmp_val*w_sec(k)); - w2_2.set(condition, tmp_val*w_sec(k)); - } + Spack Skew_w, w1_1, w1_2, w2_1, w2_2, a; + shoc_assumed_pdf_vv_parameters(w_first, w2sec, w3var, w_tol_sqd, Skew_w, w1_1, w1_2, w2_1, w2_2, a); // Find parameters for thetal - Spack thl1_1(thl_first), thl1_2(thl_first), thl2_1(0), thl2_2(0), - sqrtthl2_1(0), sqrtthl2_2(0); - { - const Smask condition = thlsec > (thl_tol*thl_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; - - const Spack corrtest1 = ekat::max(-1, ekat::min(1, wthlsec/(sqrtw2*sqrtthl))); - const Spack tmp_val_1(-corrtest1/w1_1), tmp_val_2(-corrtest1/w1_2); - - Spack Skew_thl(0); - if (dothetal_skew == true) { - const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); - Skew_thl.set(tsign>sp(0.4), sp(1.2)*Skew_w); - Skew_thl.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); - } - - if (condition.any()) { - thl2_1.set(condition, - ekat::min(100, - ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1-a)*ekat::square(tmp_val_1)) - - (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*a*(tmp_val_1 - tmp_val_2))))*thlsec); - thl2_2.set(condition, - ekat::min(100, - ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) - - (1 - a)*ekat::square(tmp_val_1)) - + (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*thlsec); - - thl1_1.set(condition, tmp_val_2*sqrtthl+thl_first); - thl1_2.set(condition, tmp_val_1*sqrtthl+thl_first); - - sqrtthl2_1.set(condition, ekat::sqrt(thl2_1)); - sqrtthl2_2.set(condition, ekat::sqrt(thl2_2)); - } - - } + Spack thl1_1, thl1_2, thl2_1, thl2_2, sqrtthl2_1, sqrtthl2_2; + shoc_assumed_pdf_thl_parameters(wthlsec,sqrtw2,sqrtthl,thlsec,thl_first,w1_1,w1_2,Skew_w,a, + thl_tol, w_thresh, + thl1_1, thl1_2, thl2_1, thl2_2, sqrtthl2_1, sqrtthl2_2); // Find parameters for total water mixing ratio - Spack qw1_1(qw_first), qw1_2(qw_first), qw2_1(0), qw2_2(0), - sqrtqw2_1(0), sqrtqw2_2(0); - { - const Smask condition = qwsec > (rt_tol*rt_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; - - const Spack corrtest2 = ekat::max(-1, ekat::min(1, wqwsec/(sqrtw2*sqrtqt))); - const Spack tmp_val_1(-corrtest2/w1_1), tmp_val_2(-corrtest2/w1_2); - - const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); - Spack Skew_qw(0); - Skew_qw.set(tsign>sp(0.4), sp(1.2)*Skew_w); - Skew_qw.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); - - if (condition.any()) { - qw2_1.set(condition, - ekat::min(100, - ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) - - (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*a*(tmp_val_1 - tmp_val_2))))*qwsec); - qw2_2.set(condition, - ekat::min(100, - ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) - + (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) - /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*qwsec); - } - - qw1_1.set(condition, tmp_val_2*sqrtqt+qw_first); - qw1_2.set(condition, tmp_val_1*sqrtqt+qw_first); - - sqrtqw2_1.set(condition, ekat::sqrt(qw2_1)); - sqrtqw2_2.set(condition, ekat::sqrt(qw2_2)); - } + Spack qw1_1, qw1_2, qw2_1, qw2_2, sqrtqw2_1, sqrtqw2_2; + shoc_assumed_pdf_qw_parameters(wqwsec, sqrtw2, Skew_w, sqrtqt, qwsec, w1_2, w1_1, qw_first, a, + rt_tol, w_thresh, + qw1_1, qw1_2, qw2_1, qw2_2, sqrtqw2_1, sqrtqw2_2); // Convert from tilde variables to "real" variables - w1_1 *= sqrtw2; - w1_1 += w_first; - w1_2 *= sqrtw2; - w1_2 += w_first; + shoc_assumed_pdf_tilde_to_real(w_first, sqrtw2, w1_1); + shoc_assumed_pdf_tilde_to_real(w_first, sqrtw2, w1_2); // Find within-plume correlations. - Spack r_qwthl_1(0); - { - const Spack testvar = a*sqrtqw2_1*sqrtthl2_1 + (1 - a)*sqrtqw2_2*sqrtthl2_2; - const auto testvar_ne_zero = testvar != 0; - if (testvar_ne_zero.any()) { - r_qwthl_1.set(testvar_ne_zero, - ekat::max(-1, - ekat::min(1, (qwthlsec - a*(qw1_1 - qw_first)*(thl1_1 - thl_first) - - (1 - a)*(qw1_2 - qw_first)*(thl1_2 - thl_first))/testvar))); - } - } + Spack r_qwthl_1; + shoc_assumed_pdf_inplume_correlations(sqrtqw2_1, sqrtthl2_1, a, sqrtqw2_2, sqrtthl2_2, qwthlsec, + qw1_1, qw_first, thl1_1, thl_first, qw1_2, thl1_2, + r_qwthl_1); // Begin to compute cloud property statistics - Spack Tl1_1 = thl1_1/(ekat::pow(basepres/pval,(rair/cp))); - Spack Tl1_2 = thl1_2/(ekat::pow(basepres/pval,(rair/cp))); + Spack Tl1_1, Tl1_2; + shoc_assumed_pdf_compute_temperature(thl1_1,pval, Tl1_1); + shoc_assumed_pdf_compute_temperature(thl1_2,pval, Tl1_2); const auto index_range = ekat::range(k*Spack::n); const Smask active_entries = (index_range < nlev); @@ -281,121 +200,54 @@ void Functions::shoc_assumed_pdf( } // Compute qs and beta - Spack qs1(0), qs2(0), beta1(0), beta2(0); - { - // Compute MurphyKoop_svp - const int liquid = 0; - const Spack esval1_1 = scream::physics::Functions::MurphyKoop_svp(Tl1_1,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_1)"); - const Spack esval1_2 = scream::physics::Functions::MurphyKoop_svp(Tl1_2,liquid,active_entries,"shoc::shoc_assumed_pdf (Tl1_2)"); - const Spack lstarn(lcond); - - qs1 = sp(0.622)*esval1_1/ekat::max(esval1_1, pval - esval1_1); - beta1 = (rair/rv)*(lstarn/(rair*Tl1_1))*(lstarn/(cp*Tl1_1)); - - // Only compute qs2 and beta2 if the two plumes are not equal - const Smask condition = (Tl1_1 != Tl1_2); - qs2 = qs1; - beta2 = beta1; - - qs2.set(condition, sp(0.622)*esval1_2/ekat::max(esval1_2, pval - esval1_2)); - beta2.set(condition, (rair/rv)*(lstarn/(rair*Tl1_2))*(lstarn/(cp*Tl1_2))); - } + Spack qs1, qs2, beta1, beta2; + shoc_assumed_pdf_compute_qs(Tl1_1, Tl1_2, pval, active_entries, qs1, beta1, qs2, beta2); // Cloud computations // Compute s terms. - Spack s1(0), std_s1(0), qn1(0), C1(0), ql1(0), - s2(0), std_s2(0), qn2(0), C2(0), ql2(0); - { - const Scalar sqrt2(std::sqrt(Scalar(2.0))), sqrt2pi(std::sqrt(2*pi)); - - // First plume - const Spack cthl1=((1 + beta1*qw1_1)/ekat::square(1 + beta1*qs1))*(cp/lcond)* - beta1*qs1*ekat::pow(pval/basepres, (rair/cp)); - const Spack cqt1 = 1/(1 + beta1*qs1); - - std_s1 = ekat::sqrt(ekat::max(0, - ekat::square(cthl1)*thl2_1 - + ekat::square(cqt1)*qw2_1 - 2*cthl1*sqrtthl2_1*cqt1*sqrtqw2_1*r_qwthl_1)); - const auto std_s1_not_small = std_s1 > std::sqrt(std::numeric_limits::min()) * 100; - s1 = qw1_1-qs1*((1 + beta1*qw1_1)/(1 + beta1*qs1)); - if (std_s1_not_small.any()) { - C1.set(std_s1_not_small, sp(0.5)*(1 + ekat::erf(s1/(sqrt2*std_s1)))); - } - C1.set(!std_s1_not_small && s1 > 0, 1); - const auto std_s1_C1_not_small = std_s1_not_small && C1 != 0; - if (std_s1_C1_not_small.any()) { - qn1.set(std_s1_C1_not_small, s1*C1+(std_s1/sqrt2pi)*ekat::exp(-sp(0.5)*ekat::square(s1/std_s1))); - } - qn1.set(!std_s1_not_small && s1 > 0, s1); - - // Checking to prevent empty clouds - const auto qn1_le_zero = qn1 <= 0; - C1.set(qn1_le_zero,0); - qn1.set(qn1_le_zero,0); - - ql1 = ekat::min(qn1, qw1_1); - - // Second plume - // Only compute variables of the second plume if the two plumes are not equal - const Smask equal(qw1_1==qw1_2 && thl2_1==thl2_2 && qs1==qs2); - std_s2.set(equal, std_s1); - s2.set(equal, s1); - C2.set(equal, C1); - qn2.set(equal, qn1); - - const Spack cthl2(!equal, - ((1 + beta2*qw1_2)/ekat::square(1 + beta2*qs2))*(cp/lcond)* - beta2*qs2*ekat::pow(pval/basepres, (rair/cp))); - const Spack cqt2(!equal, - 1/(1 + beta2*qs2)); - - const auto nequal = !equal; - if (nequal.any()) { - std_s2.set(nequal, - ekat::sqrt(ekat::max(0, - ekat::square(cthl2)*thl2_2 - + ekat::square(cqt2)*qw2_2 - 2*cthl2*sqrtthl2_2*cqt2*sqrtqw2_2*r_qwthl_1))); - s2.set(nequal, qw1_2-qs2*((1 + beta2*qw1_2)/(1 + beta2*qs2))); - const auto std_s2_not_small = std_s2 > std::sqrt(std::numeric_limits::min()) * 100; - const auto nequal_std_s2_not_small = nequal && std_s2_not_small; - if (nequal_std_s2_not_small.any()) { - C2.set(nequal_std_s2_not_small, sp(0.5)*(1 + ekat::erf(s2/(sqrt2*std_s2)))); - } - C2.set(nequal && !std_s2_not_small && s2 > 0, 1); - const auto nequal_std_s2_C2_not_small = nequal_std_s2_not_small && C2 != 0; - if (nequal_std_s2_C2_not_small.any()) { - qn2.set(nequal_std_s2_C2_not_small, s2*C2+(std_s2/sqrt2pi)*ekat::exp(-sp(0.5)*ekat::square(s2/std_s2))); - } - qn2.set(nequal && !std_s2_not_small && s2 > 0, s2); - } - - // Checking to prevent empty clouds - const auto qn2_le_zero = qn2 <= 0; - C2.set(qn2_le_zero,0); - qn2.set(qn2_le_zero,0); - - ql2 = ekat::min(qn2, qw1_2); + Spack s1, std_s1, qn1, C1, ql1, s2, std_s2, qn2, C2, ql2; + + // First plume + shoc_assumed_pdf_compute_s(qw1_1, qs1, beta1, pval, thl2_1, qw2_1, + sqrtthl2_1, sqrtqw2_1, r_qwthl_1, + s1, std_s1, qn1, C1); + + // Second plume + // Only compute variables of the second plume if the two plumes are not equal + const Smask equal(qw1_1==qw1_2 && thl2_1==thl2_2 && qs1==qs2); + std_s2.set(equal, std_s1); + s2.set(equal, s1); + C2.set(equal, C1); + qn2.set(equal, qn1); + + const auto nequal = !equal; + if (nequal.any()) { + shoc_assumed_pdf_compute_s(qw1_2, qs2, beta2, pval, thl2_2, qw2_2, + sqrtthl2_2, sqrtqw2_2, r_qwthl_1, + s2, std_s2, qn2, C2); } + ql1 = ekat::min(qn1, qw1_1); + ql2 = ekat::min(qn2, qw1_2); + // Compute SGS cloud fraction shoc_cldfrac(k) = ekat::min(1, a*C1 + (1 - a)*C2); // Compute SGS liquid water mixing ratio - shoc_ql(k) = ekat::max(0, a*ql1 + (1 - a)*ql2); + shoc_assumed_pdf_compute_sgs_liquid(a, ql1, ql2, shoc_ql(k)); // Compute cloud liquid variance (CLUBB formulation, adjusted to SHOC parameters based) - shoc_ql2(k) = ekat::max(0, a*(s1*ql1 + C1*ekat::square(std_s1)) - + (1 - a)*(s2*ql2 + C2*ekat::square(std_s2)) - - ekat::square(shoc_ql(k))); + shoc_assumed_pdf_compute_cloud_liquid_variance(a, s1, ql1, C1, std_s1, + s2, ql2, C2, std_s2, shoc_ql(k), + shoc_ql2(k)); // Compute liquid water flux - wqls(k) = a*((w1_1 - w_first)*ql1) + (1 - a)*((w1_2 - w_first)*ql2); + shoc_assumed_pdf_compute_liquid_water_flux(a, w1_1, w_first, ql1, w1_2, ql2, wqls(k)); // Compute the SGS buoyancy flux - wthv_sec(k) = wthlsec + ((1 - epsterm)/epsterm)*basetemp*wqwsec - + ((lcond/cp)*ekat::pow(basepres/pval, (rair/cp)) - - (1/epsterm)*basetemp)*wqls(k); + shoc_assumed_pdf_compute_buoyancy_flux(wthlsec, wqwsec, pval, wqls(k), + wthv_sec(k)); }); // Release temporary variables from the workspace diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp new file mode 100644 index 000000000000..600af5a6f51b --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_inplume_correlations_impl.hpp @@ -0,0 +1,50 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_INPLUME_CORRELATIONS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_INPLUME_CORRELATIONS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_inplume_correlations. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find within-plume correlation + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_inplume_correlations( + const Spack& sqrtqw2_1, + const Spack& sqrtthl2_1, + const Spack& a, + const Spack& sqrtqw2_2, + const Spack& sqrtthl2_2, + const Spack& qwthlsec, + const Spack& qw1_1, + const Spack& qw_first, + const Spack& thl1_1, + const Spack& thl_first, + const Spack& qw1_2, + const Spack& thl1_2, + Spack& r_qwthl_1) +{ + r_qwthl_1 = 0; + + const Spack testvar = a*sqrtqw2_1*sqrtthl2_1 + (1 - a)*sqrtqw2_2*sqrtthl2_2; + const auto testvar_ne_zero = testvar != 0; + if (testvar_ne_zero.any()) { + r_qwthl_1.set(testvar_ne_zero, + ekat::max(-1, + ekat::min(1, (qwthlsec - a*(qw1_1 - qw_first)*(thl1_1 - thl_first) + - (1 - a)*(qw1_2 - qw_first)*(thl1_2 - thl_first))/testvar))); + } +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp new file mode 100644 index 000000000000..d4cb020a7fa2 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_qw_parameters_impl.hpp @@ -0,0 +1,79 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_QW_PARAMETERS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_QW_PARAMETERS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_qw_parameters. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find parameters for total water mixing ratio + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_qw_parameters( + const Spack& wqwsec, + const Spack& sqrtw2, + const Spack& Skew_w, + const Spack& sqrtqt, + const Spack& qwsec, + const Spack& w1_2, + const Spack& w1_1, + const Spack& qw_first, + const Spack& a, + const Scalar rt_tol, + const Scalar w_thresh, + Spack& qw1_1, + Spack& qw1_2, + Spack& qw2_1, + Spack& qw2_2, + Spack& sqrtqw2_1, + Spack& sqrtqw2_2) +{ + qw1_1 = qw_first; + qw1_2 = qw_first; + qw2_1 = 0; + qw2_2 = 0; + sqrtqw2_1 = 0; + sqrtqw2_2 = 0; + + const Smask condition = qwsec > (rt_tol*rt_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; + + const Spack corrtest2 = ekat::max(-1, ekat::min(1, wqwsec/(sqrtw2*sqrtqt))); + const Spack tmp_val_1(-corrtest2/w1_1), tmp_val_2(-corrtest2/w1_2); + + const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); + Spack Skew_qw(0); + Skew_qw.set(tsign>sp(0.4), sp(1.2)*Skew_w); + Skew_qw.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); + + if (condition.any()) { + qw2_1.set(condition, + ekat::min(100, + ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) + - (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*a*(tmp_val_1 - tmp_val_2))))*qwsec); + qw2_2.set(condition, + ekat::min(100, + ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) - (1 - a)*ekat::square(tmp_val_1)) + + (Skew_qw - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*qwsec); + } + + qw1_1.set(condition, tmp_val_2*sqrtqt+qw_first); + qw1_2.set(condition, tmp_val_1*sqrtqt+qw_first); + + sqrtqw2_1.set(condition, ekat::sqrt(qw2_1)); + sqrtqw2_2.set(condition, ekat::sqrt(qw2_2)); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp new file mode 100644 index 000000000000..686786d6fe21 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_thl_parameters_impl.hpp @@ -0,0 +1,82 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_THL_PARAMETERS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_THL_PARAMETERS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_thl_parameters. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find parameters for thetal + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_thl_parameters( + const Spack& wthlsec, + const Spack& sqrtw2, + const Spack& sqrtthl, + const Spack& thlsec, + const Spack& thl_first, + const Spack& w1_1, + const Spack& w1_2, + const Spack& Skew_w, + const Spack& a, + const Scalar thl_tol, + const Scalar w_thresh, + Spack& thl1_1, + Spack& thl1_2, + Spack& thl2_1, + Spack& thl2_2, + Spack& sqrtthl2_1, + Spack& sqrtthl2_2) +{ + thl1_1 = thl_first; + thl1_2 = thl_first; + thl2_1 = 0; + thl2_2 = 0; + sqrtthl2_1 = 0; + sqrtthl2_2 = 0; + + const Smask condition = thlsec > (thl_tol*thl_tol) && ekat::abs(w1_2 - w1_1) > w_thresh; + + const Spack corrtest1 = ekat::max(-1, ekat::min(1, wthlsec/(sqrtw2*sqrtthl))); + const Spack tmp_val_1(-corrtest1/w1_1), tmp_val_2(-corrtest1/w1_2); + + Spack Skew_thl(0); + if (SC::dothetal_skew == true) { + const auto tsign = ekat::abs(tmp_val_1 - tmp_val_2); + Skew_thl.set(tsign>sp(0.4), sp(1.2)*Skew_w); + Skew_thl.set(tsign>sp(0.2) && tsign<=sp(0.4), (((sp(1.2)*Skew_w)/sp(0.2))*(tsign-sp(0.2)))); + } + + if (condition.any()) { + thl2_1.set(condition, + ekat::min(100, + ekat::max(0, (3*tmp_val_1*(1 - a*ekat::square(tmp_val_2) - (1-a)*ekat::square(tmp_val_1)) + - (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*a*(tmp_val_1 - tmp_val_2))))*thlsec); + thl2_2.set(condition, + ekat::min(100, + ekat::max(0, (-3*tmp_val_2*(1 - a*ekat::square(tmp_val_2) + - (1 - a)*ekat::square(tmp_val_1)) + + (Skew_thl - a*ekat::cube(tmp_val_2) - (1 - a)*ekat::cube(tmp_val_1))) + /(3*(1 - a)*(tmp_val_1 - tmp_val_2))))*thlsec); + + thl1_1.set(condition, tmp_val_2*sqrtthl+thl_first); + thl1_2.set(condition, tmp_val_1*sqrtthl+thl_first); + + sqrtthl2_1.set(condition, ekat::sqrt(thl2_1)); + sqrtthl2_2.set(condition, ekat::sqrt(thl2_2)); + } +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp new file mode 100644 index 000000000000..e2de2705fb8e --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_tilde_to_real_impl.hpp @@ -0,0 +1,32 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_TILDE_TO_REAL_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_TILDE_TO_REAL_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc_assumed_pdf_tilde_to_real. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Convert tilde variables to "real" variables + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_tilde_to_real( + const Spack& w_first, + const Spack& sqrtw2, + Spack& w1) +{ + w1 *= sqrtw2; + w1 += w_first; +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp new file mode 100644 index 000000000000..6cd237567597 --- /dev/null +++ b/components/eamxx/src/physics/shoc/impl/shoc_assumed_pdf_vv_parameters_impl.hpp @@ -0,0 +1,61 @@ +#ifndef SHOC_SHOC_ASSUMED_PDF_VV_PARAMETERS_IMPL_HPP +#define SHOC_SHOC_ASSUMED_PDF_VV_PARAMETERS_IMPL_HPP + +#include "shoc_functions.hpp" // for ETI only but harmless for GPU + +#include + +namespace scream { +namespace shoc { + +/* + * Implementation of shoc shoc_assumed_pdf_vv_parameters. Clients should NOT + * #include this file, but include shoc_functions.hpp instead. + * + * Find parameters for vertical velocity + */ + +template +KOKKOS_INLINE_FUNCTION +void Functions::shoc_assumed_pdf_vv_parameters( + const Spack& w_first, + const Spack& w_sec, + const Spack& w3var, + const Scalar w_tol_sqd, + Spack& Skew_w, + Spack& w1_1, + Spack& w1_2, + Spack& w2_1, + Spack& w2_2, + Spack& a) +{ + Skew_w = 0; + w1_1 = w_first; + w1_2 = w_first; + w2_1 = 0; + w2_2 = 0; + a = 0.5; + + const Smask condition = w_sec > w_tol_sqd; + + const Scalar tmp_val(0.4); + const Scalar one_m_tmp_val(1 - tmp_val); + const Scalar sqrtw2t(std::sqrt(1-tmp_val)); + + Skew_w.set(condition, w3var/ekat::sqrt(ekat::cube(w_sec))); + a.set(condition, + ekat::max(sp(0.01), + ekat::min(sp(0.99), + sp(0.5)*(1 - Skew_w*ekat::sqrt(1/(4*(one_m_tmp_val*one_m_tmp_val*one_m_tmp_val) + + ekat::square(Skew_w))))))); + + w1_1.set(condition, ekat::sqrt((1 - a)/a)*sqrtw2t); + w1_2.set(condition, -1*ekat::sqrt(a/(1 - a))*sqrtw2t); + w2_1.set(condition, tmp_val*w_sec); + w2_2.set(condition, tmp_val*w_sec); +} + +} // namespace shoc +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp index 120ed505f261..a41868478b8f 100644 --- a/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp +++ b/components/eamxx/src/physics/shoc/impl/shoc_main_impl.hpp @@ -61,7 +61,7 @@ Int Functions::shoc_init( return host_view(0); } -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS template KOKKOS_FUNCTION void Functions::shoc_main_internal( @@ -118,6 +118,8 @@ void Functions::shoc_main_internal( const uview_1d& shoc_ql, // Output Variables Scalar& pblh, + Scalar& ustar, + Scalar& obklen, const uview_1d& shoc_ql2, const uview_1d& tkh, // Diagnostic Output Variables @@ -144,9 +146,9 @@ void Functions::shoc_main_internal( {&rho_zt, &shoc_qv, &shoc_tabs, &dz_zt, &dz_zi}); // Local scalars - Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, - se_a{0}, ke_a{0}, wv_a{0}, wl_a{0}, - ustar{0}, kbfs{0}, obklen{0}, ustar2{0}, wstar{0}; + Scalar se_b{0}, ke_b{0}, wv_b{0}, wl_b{0}, + se_a{0}, ke_a{0}, wv_a{0}, wl_a{0}, + kbfs{0}, ustar2{0}, wstar{0}; // Scalarize some views for single entry access const auto s_thetal = ekat::scalarize(thetal); @@ -371,6 +373,8 @@ void Functions::shoc_main_internal( const view_2d& shoc_ql, // Output Variables const view_1d& pblh, + const view_1d& ustar, + const view_1d& obklen, const view_2d& shoc_ql2, const view_2d& tkh, // Diagnostic Output Variables @@ -397,9 +401,7 @@ void Functions::shoc_main_internal( const view_1d& ke_a, const view_1d& wv_a, const view_1d& wl_a, - const view_1d& ustar, const view_1d& kbfs, - const view_1d& obklen, const view_1d& ustar2, const view_1d& wstar, const view_2d& rho_zt, @@ -584,7 +586,7 @@ Int Functions::shoc_main( const SHOCInputOutput& shoc_input_output, // Input/Output const SHOCOutput& shoc_output, // Output const SHOCHistoryOutput& shoc_history_output // Output (diagnostic) -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , const SHOCTemporaries& shoc_temporaries // Temporaries for small kernels #endif ) @@ -606,7 +608,7 @@ Int Functions::shoc_main( const Scalar Ckh = shoc_runtime.Ckh; const Scalar Ckm = shoc_runtime.Ckm; -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS using ExeSpace = typename KT::ExeSpace; // SHOC main loop @@ -625,6 +627,8 @@ Int Functions::shoc_main( const Scalar vw_sfc_s{shoc_input.vw_sfc(i)}; const Scalar phis_s{shoc_input.phis(i)}; Scalar pblh_s{0}; + Scalar ustar_s{0}; + Scalar obklen_s{0}; const auto zt_grid_s = ekat::subview(shoc_input.zt_grid, i); const auto zi_grid_s = ekat::subview(shoc_input.zi_grid, i); @@ -676,12 +680,14 @@ Int Functions::shoc_main( host_dse_s, tke_s, thetal_s, qw_s, u_wind_s, v_wind_s, // Input/Output wthv_sec_s, qtracers_s, tk_s, shoc_cldfrac_s, // Input/Output shoc_ql_s, // Input/Output - pblh_s, shoc_ql2_s, tkh_s, // Output + pblh_s, ustar_s, obklen_s, shoc_ql2_s, tkh_s, // Output shoc_mix_s, w_sec_s, thl_sec_s, qw_sec_s, qwthl_sec_s, // Diagnostic Output Variables wthl_sec_s, wqw_sec_s, wtke_sec_s, uw_sec_s, vw_sec_s, // Diagnostic Output Variables w3_s, wqls_sec_s, brunt_s, isotropy_s); // Diagnostic Output Variables shoc_output.pblh(i) = pblh_s; + shoc_output.ustar(i) = ustar_s; + shoc_output.obklen(i) = obklen_s; }); Kokkos::fence(); #else @@ -700,14 +706,14 @@ Int Functions::shoc_main( shoc_input_output.host_dse, shoc_input_output.tke, shoc_input_output.thetal, shoc_input_output.qw, u_wind_s, v_wind_s, // Input/Output shoc_input_output.wthv_sec, shoc_input_output.qtracers, shoc_input_output.tk, shoc_input_output.shoc_cldfrac, // Input/Output shoc_input_output.shoc_ql, // Input/Output - shoc_output.pblh, shoc_output.shoc_ql2, shoc_output.tkh, // Output + shoc_output.pblh, shoc_output.ustar, shoc_output.obklen, shoc_output.shoc_ql2, shoc_output.tkh, // Output shoc_history_output.shoc_mix, shoc_history_output.w_sec, shoc_history_output.thl_sec, shoc_history_output.qw_sec, shoc_history_output.qwthl_sec, // Diagnostic Output Variables shoc_history_output.wthl_sec, shoc_history_output.wqw_sec, shoc_history_output.wtke_sec, shoc_history_output.uw_sec, shoc_history_output.vw_sec, // Diagnostic Output Variables shoc_history_output.w3, shoc_history_output.wqls_sec, shoc_history_output.brunt, shoc_history_output.isotropy, // Diagnostic Output Variables // Temporaries shoc_temporaries.se_b, shoc_temporaries.ke_b, shoc_temporaries.wv_b, shoc_temporaries.wl_b, shoc_temporaries.se_a, shoc_temporaries.ke_a, shoc_temporaries.wv_a, shoc_temporaries.wl_a, - shoc_temporaries.ustar, shoc_temporaries.kbfs, shoc_temporaries.obklen, shoc_temporaries.ustar2, + shoc_temporaries.kbfs, shoc_temporaries.ustar2, shoc_temporaries.wstar, shoc_temporaries.rho_zt, shoc_temporaries.shoc_qv, shoc_temporaries.tabs, shoc_temporaries.dz_zt, shoc_temporaries.dz_zi); #endif diff --git a/components/eamxx/src/physics/shoc/shoc_functions.hpp b/components/eamxx/src/physics/shoc/shoc_functions.hpp index 74602c688c23..8913ad53841b 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions.hpp +++ b/components/eamxx/src/physics/shoc/shoc_functions.hpp @@ -159,6 +159,10 @@ struct Functions // planetary boundary layer depth [m] view_1d pblh; + // surface friction velocity [m/s] + view_1d ustar; + // Monin Obukhov length [m] + view_1d obklen; // cloud liquid mixing ratio variance [kg^2/kg^2] view_2d shoc_ql2; // eddy coefficient for heat [m2/s] @@ -199,7 +203,7 @@ struct Functions view_2d isotropy; }; -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS struct SHOCTemporaries { SHOCTemporaries() = default; @@ -211,9 +215,7 @@ struct Functions view_1d ke_a; view_1d wv_a; view_1d wl_a; - view_1d ustar; view_1d kbfs; - view_1d obklen; view_1d ustar2; view_1d wstar; @@ -270,7 +272,7 @@ struct Functions const uview_1d& zt_grid, const Scalar& phis, const uview_1d& host_dse); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void update_host_dse_disp( const Int& shcol, const Int& nlev, @@ -322,7 +324,7 @@ struct Functions const MemberType& team, const Int& nlev, const uview_1d& tke); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void check_tke_disp( const Int& schol, const Int& nlev, @@ -361,7 +363,7 @@ struct Functions Scalar& ke_int, Scalar& wv_int, Scalar& wl_int); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_energy_integrals_disp( const Int& shcol, const Int& nlev, @@ -407,12 +409,12 @@ struct Functions const Workspace& workspace, const uview_1d& thl_sec, const uview_1d& qw_sec, const uview_1d& wthl_sec, const uview_1d& wqw_sec, const uview_1d& qwthl_sec, const uview_1d& uw_sec, const uview_1d& vw_sec, const uview_1d& wtke_sec, const uview_1d& w_sec); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void diag_second_shoc_moments_disp( const Int& shcol, const Int& nlev, const Int& nlevi, - const Scalar& thl2tune, - const Scalar& qw2tune, - const Scalar& qwthl2tune, + const Scalar& thl2tune, + const Scalar& qw2tune, + const Scalar& qwthl2tune, const Scalar& w2tune, const view_2d& thetal, const view_2d& qw, @@ -483,7 +485,7 @@ struct Functions Scalar& ustar, Scalar& kbfs, Scalar& obklen); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_diag_obklen_disp( const Int& shcol, const Int& nlev, @@ -520,7 +522,7 @@ struct Functions const Workspace& workspace, const uview_1d& brunt, const uview_1d& shoc_mix); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_length_disp( const Int& shcol, const Int& nlev, @@ -562,7 +564,7 @@ struct Functions const uview_1d& pint, const Workspace& workspace, const uview_1d& host_dse); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_energy_fixer_disp( const Int& shcol, const Int& nlev, @@ -595,7 +597,7 @@ struct Functions const uview_1d& qw, const uview_1d& ql, const uview_1d& qv); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void compute_shoc_vapor_disp( const Int& shcol, const Int& nlev, @@ -612,7 +614,7 @@ struct Functions const uview_1d& ql, const uview_1d& inv_exner, const uview_1d& tabs); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void compute_shoc_temperature_disp( const Int& shcol, const Int& nlev, @@ -648,7 +650,7 @@ struct Functions const uview_1d& tke, const uview_1d& u_wind, const uview_1d& v_wind); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void update_prognostics_implicit_disp( const Int& shcol, const Int& nlev, @@ -695,7 +697,7 @@ struct Functions const uview_1d& zi_grid, const Workspace& workspace, const uview_1d& w3); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void diag_third_shoc_moments_disp( const Int& shcol, const Int& nlev, @@ -752,7 +754,7 @@ struct Functions const uview_1d& wqls, const uview_1d& wthv_sec, const uview_1d& shoc_ql2); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_assumed_pdf_disp( const Int& shcol, const Int& nlev, @@ -778,6 +780,154 @@ struct Functions const view_2d& shoc_ql2); #endif + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_compute_buoyancy_flux( + const Spack& wthlsec, + const Spack& wqwsec, + const Spack& pval, + const Spack& wqls, + Spack& wthv_sec); + + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_compute_cloud_liquid_variance( + const Spack& a, + const Spack& s1, + const Spack& ql1, + const Spack& C1, + const Spack& std_s1, + const Spack& s2, + const Spack& ql2, + const Spack& C2, + const Spack& std_s2, + const Spack& shoc_ql, + Spack& shoc_ql2); + + + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_compute_liquid_water_flux( + const Spack& a, + const Spack& w1_1, + const Spack& w_first, + const Spack& ql1, + const Spack& w1_2, + const Spack& ql2, + Spack& wqls); + + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_compute_qs( + const Spack& Tl1_1, + const Spack& Tl1_2, + const Spack& pval, + const Smask& active_entries, + Spack& qs1, + Spack& beta1, + Spack& qs2, + Spack& beta2); + + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_compute_s( + const Spack& qw1, + const Spack& qs, + const Spack& beta, + const Spack& pval, + const Spack& thl2, + const Spack& qw2, + const Spack& sqrtthl2, + const Spack& sqrtqw2, + const Spack& r_qwthl, + Spack& s, + Spack& std_s, + Spack& qn, + Spack& C); + + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_compute_sgs_liquid( + const Spack& a, + const Spack& ql1, + const Spack& ql2, + Spack& shoc_ql); + + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_compute_temperature( + const Spack& thl1, + const Spack& pval, + Spack& Tl1); + + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_inplume_correlations( + const Spack& sqrtqw2_1, + const Spack& sqrtthl2_1, + const Spack& a, + const Spack& sqrtqw2_2, + const Spack& sqrtthl2_2, + const Spack& qwthlsec, + const Spack& qw1_1, + const Spack& qw_first, + const Spack& thl1_1, + const Spack& thl_first, + const Spack& qw1_2, + const Spack& thl1_2, + Spack& r_qwthl_1); + + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_qw_parameters( + const Spack& wqwsec, + const Spack& sqrtw2, + const Spack& Skew_w, + const Spack& sqrtqt, + const Spack& qwsec, + const Spack& w1_2, + const Spack& w1_1, + const Spack& qw_first, + const Spack& a, + const Scalar rt_tol, + const Scalar w_thresh, + Spack& qw1_1, + Spack& qw1_2, + Spack& qw2_1, + Spack& qw2_2, + Spack& sqrtqw2_1, + Spack& sqrtqw2_2); + + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_thl_parameters( + const Spack& wthlsec, + const Spack& sqrtw2, + const Spack& sqrtthl, + const Spack& thlsec, + const Spack& thl_first, + const Spack& w1_1, + const Spack& w1_2, + const Spack& Skew_w, + const Spack& a, + const Scalar thl_tol, + const Scalar w_thresh, + Spack& thl1_1, + Spack& thl1_2, + Spack& thl2_1, + Spack& thl2_2, + Spack& sqrtthl2_1, + Spack& sqrtthl2_2); + + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_tilde_to_real( + const Spack& w_first, + const Spack& sqrtw2, + Spack& w1); + + KOKKOS_INLINE_FUNCTION + static void shoc_assumed_pdf_vv_parameters( + const Spack& w_first, + const Spack& w_sec, + const Spack& w3var, + const Scalar w_tol_sqd, + Spack& Skew_w, + Spack& w1_1, + Spack& w1_2, + Spack& w2_1, + Spack& w2_2, + Spack& a); + KOKKOS_FUNCTION static void compute_shr_prod( const MemberType& team, @@ -833,7 +983,7 @@ struct Functions const Int& ntop_shoc, const view_1d& pref_mid); -#ifndef SCREAM_SMALL_KERNELS +#ifndef SCREAM_SHOC_SMALL_KERNELS KOKKOS_FUNCTION static void shoc_main_internal( const MemberType& team, @@ -889,6 +1039,8 @@ struct Functions const uview_1d& shoc_ql, // Output Variables Scalar& pblh, + Scalar& ustar, + Scalar& obklen, const uview_1d& shoc_ql2, const uview_1d& tkh, // Diagnostic Output Variables @@ -961,6 +1113,8 @@ struct Functions const view_2d& shoc_ql, // Output Variables const view_1d& pblh, + const view_1d& ustar, + const view_1d& obklen, const view_2d& shoc_ql2, const view_2d& tkh, // Diagnostic Output Variables @@ -987,9 +1141,7 @@ struct Functions const view_1d& ke_a, const view_1d& wv_a, const view_1d& wl_a, - const view_1d& ustar, const view_1d& kbfs, - const view_1d& obklen, const view_1d& ustar2, const view_1d& wstar, const view_2d& rho_zt, @@ -1014,7 +1166,7 @@ struct Functions const SHOCInputOutput& shoc_input_output, // Input/Output const SHOCOutput& shoc_output, // Output const SHOCHistoryOutput& shoc_history_output // Output (diagnostic) -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , const SHOCTemporaries& shoc_temporaries // Temporaries for small kernels #endif ); @@ -1085,7 +1237,7 @@ struct Functions const uview_1d& cldn, const Workspace& workspace, Scalar& pblh); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void pblintd_disp( const Int& shcol, const Int& nlev, @@ -1117,7 +1269,7 @@ struct Functions const uview_1d& dz_zt, const uview_1d& dz_zi, const uview_1d& rho_zt); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_grid_disp( const Int& shcol, const Int& nlev, @@ -1175,7 +1327,7 @@ struct Functions const uview_1d& tk, const uview_1d& tkh, const uview_1d& isotropy); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS static void shoc_tke_disp( const Int& shcol, const Int& nlev, @@ -1260,4 +1412,19 @@ struct Functions #endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE +// Some functions should be inlined, thus do not use ETI +# include "shoc_assumed_pdf_compute_buoyancy_flux_impl.hpp" +# include "shoc_assumed_pdf_compute_cloud_liquid_variance_impl.hpp" +# include "shoc_assumed_pdf_compute_liquid_water_flux_impl.hpp" +# include "shoc_assumed_pdf_compute_qs_impl.hpp" +# include "shoc_assumed_pdf_compute_s_impl.hpp" +# include "shoc_assumed_pdf_compute_sgs_liquid_impl.hpp" +# include "shoc_assumed_pdf_compute_temperature_impl.hpp" +# include "shoc_assumed_pdf_inplume_correlations_impl.hpp" +# include "shoc_assumed_pdf_qw_parameters_impl.hpp" +# include "shoc_assumed_pdf_compute_s_impl.hpp" +# include "shoc_assumed_pdf_thl_parameters_impl.hpp" +# include "shoc_assumed_pdf_tilde_to_real_impl.hpp" +# include "shoc_assumed_pdf_vv_parameters_impl.hpp" + #endif // SHOC_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp index dc45e2456aa8..e7468062d3c5 100644 --- a/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp +++ b/components/eamxx/src/physics/shoc/shoc_functions_f90.cpp @@ -2801,7 +2801,9 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, uw_sfc_d (temp_1d_d[index_counter++]), vw_sfc_d (temp_1d_d[index_counter++]), phis_d (temp_1d_d[index_counter++]), - pblh_d ("pblh",shcol); + pblh_d ("pblh",shcol), + ustar_d ("ustar",shcol), + obklen_d ("obklen",shcol); index_counter = 0; view_2d @@ -2878,7 +2880,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, SHF::SHOCInputOutput shoc_input_output{host_dse_d, tke_d, thetal_d, qw_d, horiz_wind_d, wthv_sec_d, qtracers_cxx_d, tk_d, shoc_cldfrac_d, shoc_ql_d}; - SHF::SHOCOutput shoc_output{pblh_d, shoc_ql2_d, tkh_d}; + SHF::SHOCOutput shoc_output{pblh_d, ustar_d, obklen_d, shoc_ql2_d, tkh_d}; SHF::SHOCHistoryOutput shoc_history_output{shoc_mix_d, w_sec_d, thl_sec_d, qw_sec_d, qwthl_sec_d, wthl_sec_d, wqw_sec_d, wtke_sec_d, uw_sec_d, vw_sec_d, w3_d, wqls_sec_d, @@ -2887,7 +2889,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, const auto nlevi_packs = ekat::npack(nlevi); -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS view_1d se_b ("se_b", shcol), ke_b ("ke_b", shcol), @@ -2897,9 +2899,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, ke_a ("ke_a", shcol), wv_a ("wv_a", shcol), wl_a ("wl_a", shcol), - ustar ("ustar", shcol), kbfs ("kbfs", shcol), - obklen ("obklen", shcol), ustar2 ("ustar2", shcol), wstar ("wstar", shcol); @@ -2911,7 +2911,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, dz_zi ("dz_zi", shcol, nlevi_packs); SHF::SHOCTemporaries shoc_temporaries{ - se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, ustar, kbfs, obklen, ustar2, wstar, + se_b, ke_b, wv_b, wl_b, se_a, ke_a, wv_a, wl_a, kbfs, ustar2, wstar, rho_zt, shoc_qv, tabs, dz_zt, dz_zi}; #endif @@ -2923,7 +2923,7 @@ Int shoc_main_f(Int shcol, Int nlev, Int nlevi, Real dtime, Int nadv, Int npbl, const auto elapsed_microsec = SHF::shoc_main(shcol, nlev, nlevi, npbl, nadv, num_qtracers, dtime, workspace_mgr, shoc_runtime_options, shoc_input, shoc_input_output, shoc_output, shoc_history_output -#ifdef SCREAM_SMALL_KERNELS +#ifdef SCREAM_SHOC_SMALL_KERNELS , shoc_temporaries #endif ); diff --git a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt index 87e55959c528..f61de7d76a4c 100644 --- a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt @@ -78,7 +78,7 @@ if (NOT SCREAM_ONLY_GENERATE_BASELINES) THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} ) - if (NOT SCREAM_SMALL_KERNELS) + if (NOT SCREAM_SHOC_SMALL_KERNELS) CreateUnitTest(shoc_sk_tests "${SHOC_TESTS_SRCS}" LIBS shoc_sk THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} diff --git a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp index 3f1ab03f1a03..e947946fa203 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_assumed_pdf_tests.cpp @@ -102,8 +102,6 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } // Check that the inputs make sense - - // Load input data for(Int s = 0; s < shcol; ++s) { for(Int n = 0; n < nlev; ++n) { const auto offset = n + s * nlev; @@ -142,8 +140,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { // For this test we want exactly two columns REQUIRE(SDS.shcol == 2); - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Verify the result // Make sure cloud fraction is either 1 or 0 and all @@ -156,9 +160,8 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { REQUIRE( (SDS.shoc_cldfrac[offset] == 0 || SDS.shoc_cldfrac[offset] == 1) ); REQUIRE(SDS.wqls[offset] == 0); REQUIRE(SDS.wthv_sec[offset] == 0); - REQUIRE(SDS.shoc_ql2[offset] == 0); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 REQUIRE(SDS.shoc_ql[offset] >= 0); - } } @@ -182,8 +185,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } } - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Verify the result // Make sure cloud fraction is either 1 or 0 and all @@ -197,7 +206,7 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { REQUIRE(SDS.wqls[offset] == 0); REQUIRE(SDS.wthv_sec[offset] != 0); REQUIRE(std::abs(SDS.wthv_sec[offset] < wthv_sec_bound)); - REQUIRE(SDS.shoc_ql2[offset] == 0); + REQUIRE(std::abs(SDS.shoc_ql2[offset]) < std::numeric_limits::epsilon()); // Computation is not exactly BFB with 0 REQUIRE(SDS.shoc_ql[offset] >= 0); REQUIRE(SDS.shoc_ql[offset] < shoc_ql_bound); } @@ -240,8 +249,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } } - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Check the result @@ -324,8 +339,14 @@ struct UnitWrap::UnitTest::TestShocAssumedPdf { } } - // Call the fortran implementation - shoc_assumed_pdf(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + shoc_assumed_pdf_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.w_field, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.w_sec, SDS.wqw_sec, + SDS.qwthl_sec, SDS.w3, SDS.pres, SDS.zt_grid, SDS.zi_grid, + SDS.shoc_cldfrac, SDS.shoc_ql, SDS.wqls, SDS.wthv_sec, SDS.shoc_ql2); + SDS.transpose(); // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp index 4c9ff408e675..3018d0e5aa6a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_brunt_length_tests.cpp @@ -90,8 +90,11 @@ struct UnitWrap::UnitTest::TestCompBruntShocLength { } } - // Call the fortran implementation - compute_brunt_shoc_length(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + compute_brunt_shoc_length_f(SDS.nlev,SDS.nlevi,SDS.shcol,SDS.dz_zt,SDS.thv,SDS.thv_zi,SDS.brunt); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp index 1069b51d408b..bd023e2ed794 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_check_length_tests.cpp @@ -75,8 +75,11 @@ struct UnitWrap::UnitTest::TestCheckShocLength { REQUIRE(SDS.host_dy[s] > 0); } - // Call the fortran implementation - check_length_scale_shoc_length(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + check_length_scale_shoc_length_f(SDS.nlev,SDS.shcol,SDS.host_dx,SDS.host_dy,SDS.shoc_mix); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp index ca22ffe3b2ef..75581e2283ce 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_check_tke_tests.cpp @@ -52,8 +52,11 @@ struct UnitWrap::UnitTest::TestShocCheckTke { // Check some input REQUIRE((SDS.shcol > 0 && SDS.nlev > 0)); - // call the fortran implementation - check_tke(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + check_tke_f(SDS.nlev, SDS.shcol, SDS.tke); + SDS.transpose(); // Check the result against the input values for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp index ee8eb4df9523..28bef1aae259 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_clip_third_moms_tests.cpp @@ -82,8 +82,11 @@ struct UnitWrap::UnitTest::TestClipThirdMoms { REQUIRE(w3_large == true); } - // Call the fortran implementation - clipping_diag_third_shoc_moments(SDS); + // Call the C++ implementation. + SDS.transpose(); + // expects data in fortran layout + clipping_diag_third_shoc_moments_f(SDS.nlevi,SDS.shcol,SDS.w_sec_zi,SDS.w3); + SDS.transpose(); // Check the result // For large values of w3, verify that the result has been reduced diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp index 0e58b658d475..d63bf34bb50e 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_diag_third_tests.cpp @@ -145,8 +145,15 @@ struct UnitWrap::UnitTest::TestShocCompDiagThird { } } - // Call the fortran implementation - compute_diag_third_shoc_moment(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + compute_diag_third_shoc_moment_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, + SDS.wthl_sec,SDS.tke,SDS.dz_zt, + SDS.dz_zi,SDS.isotropy_zi, + SDS.brunt_zi,SDS.w_sec_zi,SDS.thetal_zi, + SDS.w3); + SDS.transpose(); // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp index ae359978cf32..27452cdcf703 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_temperature_tests.cpp @@ -67,8 +67,10 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } } - // Call the fortran implementation - compute_shoc_temperature(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); + SDS.transpose(); // go back to C layout // Require that absolute temperature is equal to thetal for(Int s = 0; s < shcol; ++s) { @@ -113,8 +115,10 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } } - // Call the fortran implementation - compute_shoc_temperature(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); + SDS.transpose(); // go back to C layout // Require that absolute temperature is greather than thetal for(Int s = 0; s < shcol; ++s) { @@ -172,8 +176,10 @@ struct UnitWrap::UnitTest::TestComputeShocTemp { } } - // Call the fortran implementation - compute_shoc_temperature(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_temperature_f(SDS.shcol, SDS.nlev, SDS.thetal, SDS.ql, SDS.inv_exner, SDS.tabs); + SDS.transpose(); // go back to C layout // Require that absolute temperature be less than thetal for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp index 004f072405ba..afd184e18231 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_compute_shoc_vapor_tests.cpp @@ -62,8 +62,10 @@ struct UnitWrap::UnitTest::TestComputeShocVapor { } } - // Call the fortran implementation - compute_shoc_vapor(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shoc_vapor_f(SDS.shcol, SDS.nlev, SDS.qw, SDS.ql, SDS.qv); + SDS.transpose(); // go back to C layout // Verify the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp index 3662cc2741a0..930faf31d6b2 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_obklen_tests.cpp @@ -83,8 +83,13 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { REQUIRE( (SDS.qv_sfc[s] > 0 && SDS.qv_sfc[s] < 0.1) ); } - // Call the fortran implementation - shoc_diag_obklen(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_diag_obklen_f(SDS.shcol, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.thl_sfc, SDS.cldliq_sfc, SDS.qv_sfc, SDS.ustar, SDS.kbfs, + SDS.obklen); + SDS.transpose(); // Check the result @@ -148,8 +153,13 @@ struct UnitWrap::UnitTest::TestShocDiagObklen { SDS.wthl_sfc[s]+SDS.wqw_sfc[s]); } - // Call the fortran implementation - shoc_diag_obklen(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_diag_obklen_f(SDS.shcol, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.thl_sfc, SDS.cldliq_sfc, SDS.qv_sfc, SDS.ustar, SDS.kbfs, + SDS.obklen); + SDS.transpose(); // Verify that DIMENSIONLESS obukhov length decreases as columns // increases due to the increasing surface fluxes diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp index 507103e59a1b..25f56139d776 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_srf_test.cpp @@ -51,8 +51,8 @@ struct UnitWrap::UnitTest::TestSecondMomSrf { SDS.vw_sfc[s] = vw_sfc[s]; } - // Call the fortran implementation - diag_second_moments_srf(SDS); + // Call the C++ implementation + shoc_diag_second_moments_srf_f(SDS.shcol, SDS.wthl_sfc, SDS.uw_sfc, SDS.vw_sfc, SDS.ustar2, SDS.wstar); // Verify the output for (Int s = 0; s < shcol; ++s){ diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp index 68fccb11bfcb..d4dbd3e7b71f 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_mom_ubycond_test.cpp @@ -42,8 +42,9 @@ struct UnitWrap::UnitTest::TestSecondMomUbycond { REQUIRE(SDS.shcol == shcol); REQUIRE(shcol > 0); - // Call the fortran implementation - diag_second_moments_ubycond(SDS); + // Call the C++ implementation + shoc_diag_second_moments_ubycond_f(SDS.shcol, SDS.thl_sec, SDS.qw_sec, SDS.qwthl_sec, SDS.wthl_sec, + SDS.wqw_sec, SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec); // Verify the result // all output should be zero. diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp index bd2a3ac60ee9..1f1f3df8e5cc 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_lbycond_tests.cpp @@ -75,8 +75,11 @@ struct UnitWrap::UnitTest::TestDiagSecondMomentsLbycond { REQUIRE(SDS.ustar2[s] >= 0); } - // Call the fortran implementation - diag_second_moments_lbycond(SDS); + // Call the C++ implementation + diag_second_moments_lbycond_f(SDS.shcol, SDS.wthl_sfc, SDS.wqw_sfc, SDS.uw_sfc, + SDS.vw_sfc, SDS.ustar2, SDS.wstar, SDS.wthl_sec, + SDS.wqw_sec, SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec, + SDS.thl_sec, SDS.qw_sec, SDS.qwthl_sec); // Verify output is as expected for (Int s = 0; s < shcol; ++s){ diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp index 1ff4df42d168..f83bfa313f9c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_moments_tests.cpp @@ -162,8 +162,13 @@ struct UnitWrap::UnitTest::TestDiagSecondMoments { } } - // Call the fortran implementation - diag_second_moments(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + diag_second_moments_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.u_wind, SDS.v_wind, + SDS.tke, SDS.isotropy, SDS.tkh, SDS.tk, SDS.dz_zi, SDS.zt_grid, SDS.zi_grid, SDS.shoc_mix, + SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.qwthl_sec, SDS.uw_sec, + SDS.vw_sec, SDS.wtke_sec, SDS.w_sec); + SDS.transpose(); // go back to C layout // Verify output makes sense for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp index 86084fef344a..0385335c808b 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_second_shoc_moments_tests.cpp @@ -160,8 +160,13 @@ struct UnitWrap::UnitTest::TestDiagSecondShocMoments { } } - // Call the fortran implementation - diag_second_shoc_moments(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + diag_second_shoc_moments_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.thetal, SDS.qw, SDS.u_wind, SDS.v_wind, SDS.tke, SDS.isotropy, + SDS.tkh, SDS.tk, SDS.dz_zi, SDS.zt_grid, SDS.zi_grid, SDS.shoc_mix, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.uw_sfc, SDS.vw_sfc, SDS.thl_sec, SDS.qw_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.qwthl_sec, + SDS.uw_sec, SDS.vw_sec, SDS.wtke_sec, SDS.w_sec); + SDS.transpose(); // go back to C layout // Verify output makes sense for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp index e6430fa00d7f..82b4d88c0f70 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_diag_third_tests.cpp @@ -142,8 +142,14 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } } - // Call the fortran implementation - diag_third_shoc_moments(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + diag_third_shoc_moments_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, + SDS.wthl_sec,SDS.isotropy,SDS.brunt,SDS.thetal, + SDS.tke,SDS.dz_zt,SDS.dz_zi,SDS.zt_grid,SDS.zi_grid, + SDS.w3); + SDS.transpose(); // Check to make sure there is at least one // positive w3 value for convective boundary layer @@ -186,8 +192,14 @@ struct UnitWrap::UnitTest::TestShocDiagThird { } } - // Call the fortran implementation - diag_third_shoc_moments(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + diag_third_shoc_moments_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.w_sec,SDS.thl_sec, + SDS.wthl_sec,SDS.isotropy,SDS.brunt,SDS.thetal, + SDS.tke,SDS.dz_zt,SDS.dz_zi,SDS.zt_grid,SDS.zi_grid, + SDS.w3); + SDS.transpose(); // Verify that new result is greater or equal in magnitude // that the result from test one diff --git a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp index 4cb8d7a99c19..0a96e5adc629 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_eddy_diffusivities_tests.cpp @@ -97,8 +97,11 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } } - // Call the fortran implementation - eddy_diffusivities(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, + SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); + SDS.transpose(); // go back to C layout // Check to make sure the answers in the columns are different for(Int s = 0; s < shcol-1; ++s) { @@ -164,8 +167,11 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } } - // Call the fortran implementation - eddy_diffusivities(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, + SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); + SDS.transpose(); // go back to C layout // Check to make sure the answers in the columns are larger // when the length scale and shear term are larger @@ -234,8 +240,11 @@ struct UnitWrap::UnitTest::TestShocEddyDiff { } } - // Call the fortran implementation - eddy_diffusivities(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + eddy_diffusivities_f(SDS.nlev, SDS.shcol, SDS.pblh, SDS.zt_grid, SDS.tabs, SDS.shoc_mix, + SDS.sterm_zt, SDS.isotropy, SDS.tke, SDS.tkh, SDS.tk); + SDS.transpose(); // go back to C layout // Check to make sure the diffusivities are smaller // in the columns where isotropy and tke are smaller diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp index 8ebf2a69c88e..dfb37104f018 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_fixer_tests.cpp @@ -157,8 +157,15 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } } - // Call the fortran implementation - shoc_energy_fixer(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_energy_fixer_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, + SDS.zt_grid, SDS.zi_grid, SDS.se_b, SDS.ke_b, SDS.wv_b, + SDS.wl_b, SDS.se_a, SDS.ke_a, SDS.wv_a, SDS.wl_a, SDS.wthl_sfc, + SDS.wqw_sfc, SDS.rho_zt, SDS.tke, SDS.pint, + SDS.host_dse); + SDS.transpose(); // Check test // Verify that the dry static energy has not changed if surface @@ -231,8 +238,15 @@ struct UnitWrap::UnitTest::TestShocEnergyFixer { } } - // Call the fortran implementation - shoc_energy_fixer(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_energy_fixer_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, + SDS.zt_grid, SDS.zi_grid, SDS.se_b, SDS.ke_b, SDS.wv_b, + SDS.wl_b, SDS.se_a, SDS.ke_a, SDS.wv_a, SDS.wl_a, SDS.wthl_sfc, + SDS.wqw_sfc, SDS.rho_zt, SDS.tke, SDS.pint, + SDS.host_dse); + SDS.transpose(); // Verify the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp index 705d5ad02f25..8142da9371da 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_integral_tests.cpp @@ -107,8 +107,13 @@ struct UnitWrap::UnitTest::TestShocEnergyInt { } } - // Call the fortran implementation - shoc_energy_integrals(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_energy_integrals_f(SDS.shcol, SDS.nlev, SDS.host_dse, SDS.pdel, + SDS.rtm, SDS.rcm, SDS.u_wind, SDS.v_wind, + SDS.se_int, SDS.ke_int, SDS.wv_int, SDS.wl_int); + SDS.transpose(); // Check test for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp index 74d5094076b0..a4e01eadb852 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_energy_update_dse_tests.cpp @@ -108,8 +108,12 @@ struct UnitWrap::UnitTest::TestShocUpdateDse { } } - // Call the fortran implementation - update_host_dse(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + update_host_dse_f(SDS.shcol,SDS.nlev,SDS.thlm,SDS.shoc_ql,SDS.inv_exner,SDS.zt_grid, + SDS.phis,SDS.host_dse); + SDS.transpose(); // Check test for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp index d14fdf199175..6a79eeca02ee 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_grid_tests.cpp @@ -79,8 +79,10 @@ struct UnitWrap::UnitTest::TestShocGrid { } } - // Call the fortran implementation - shoc_grid(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + shoc_grid_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.zt_grid, SDS.zi_grid, SDS.pdel, SDS.dz_zt, SDS.dz_zi, SDS.rho_zt); + SDS.transpose(); // go back to C layout // First check that dz is correct for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp index 6b641b57d61d..53280dab17a6 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_impli_comp_tmpi_tests.cpp @@ -86,8 +86,10 @@ struct UnitWrap::UnitTest::TestImpCompTmpi { } } - // Call the fortran implementation - compute_tmpi(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_tmpi_f(SDS.nlevi, SDS.shcol, SDS.dtime, SDS.rho_zi, SDS.dz_zi, SDS.tmpi); + SDS.transpose(); // go back to C layout // Verify result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp index e2fabf24cb36..aad9c0266869 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_impli_dp_inverse_tests.cpp @@ -73,8 +73,10 @@ struct UnitWrap::UnitTest::TestImpDpInverse { } } - // Call the fortran implementation - dp_inverse(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + dp_inverse_f(SDS.nlev, SDS.shcol, SDS.rho_zt, SDS.dz_zt, SDS.rdp_zt); + SDS.transpose(); // go back to C layout // Verify result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp index dee0e30beda4..84faaebe4b91 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_l_inf_length_tests.cpp @@ -88,8 +88,11 @@ struct UnitWrap::UnitTest::TestLInfShocLength { } } - // Call the fortran implementation - compute_l_inf_shoc_length(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + compute_l_inf_shoc_length_f(SDS.nlev,SDS.shcol,SDS.zt_grid,SDS.dz_zt,SDS.tke,SDS.l_inf); + SDS.transpose(); // Check the results // Make sure result is bounded correctly diff --git a/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp index 873feffeb455..0f890db9ce11 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_length_tests.cpp @@ -120,8 +120,13 @@ struct UnitWrap::UnitTest::TestShocLength { } } - // Call the Fortran implementation - shoc_length(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + shoc_length_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.host_dx,SDS.host_dy, + SDS.zt_grid,SDS.zi_grid,SDS.dz_zt,SDS.tke, + SDS.thv,SDS.brunt,SDS.shoc_mix); + SDS.transpose(); // Verify output for(Int s = 0; s < shcol; ++s) { @@ -169,8 +174,13 @@ struct UnitWrap::UnitTest::TestShocLength { SDS.host_dy[s] = host_dy_small; } - // call fortran implentation - shoc_length(SDS); + // call C++ implentation + SDS.transpose(); + // expects data in fortran layout + shoc_length_f(SDS.shcol,SDS.nlev,SDS.nlevi,SDS.host_dx,SDS.host_dy, + SDS.zt_grid,SDS.zi_grid,SDS.dz_zt,SDS.tke, + SDS.thv,SDS.brunt,SDS.shoc_mix); + SDS.transpose(); // Verify output for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp index 2c55de1433df..0a9c666688b5 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_linear_interp_tests.cpp @@ -99,8 +99,10 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - // Call the fortran implementation - linear_interp(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + linear_interp_f(SDS.x1, SDS.x2, SDS.y1, SDS.y2, SDS.km1, SDS.km2, SDS.ncol, SDS.minthresh); + SDS.transpose(); // go back to C layout // First check that all output temperatures are greater than zero @@ -191,7 +193,10 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - linear_interp(SDS2); + // Call the C++ implementation + SDS2.transpose(); // _f expects data in fortran layout + linear_interp_f(SDS2.x1, SDS2.x2, SDS2.y1, SDS2.y2, SDS2.km1, SDS2.km2, SDS2.ncol, SDS2.minthresh); + SDS2.transpose(); // go back to C layout // Check the result, make sure output is bounded correctly @@ -275,7 +280,10 @@ struct UnitWrap::UnitTest::TestShocLinearInt { } } - linear_interp(d); + // Call the C++ implementation + d.transpose(); // _f expects data in fortran layout + linear_interp_f(d.x1, d.x2, d.y1, d.y2, d.km1, d.km2, d.ncol, d.minthresh); + d.transpose(); // go back to C layout // The combination of single-precision and randomness generating points // close together can result in larger error margins. diff --git a/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp index e699c3283284..5c3eb8f9860e 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_main_tests.cpp @@ -258,8 +258,19 @@ struct UnitWrap::UnitTest::TestShocMain { } - // Call the fortran implementation - shoc_main(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + const int npbl = shoc_init_f(SDS.nlev, SDS.pref_mid, SDS.nbot_shoc, SDS.ntop_shoc); + + shoc_main_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.nadv, npbl, SDS.host_dx, SDS.host_dy, + SDS.thv, SDS.zt_grid, SDS.zi_grid, SDS.pres, SDS.presi, SDS.pdel, SDS.wthl_sfc, + SDS.wqw_sfc, SDS.uw_sfc, SDS.vw_sfc, SDS.wtracer_sfc, SDS.num_qtracers, + SDS.w_field, SDS.inv_exner, SDS.phis, SDS.host_dse, SDS.tke, SDS.thetal, SDS.qw, + SDS.u_wind, SDS.v_wind, SDS.qtracers, SDS.wthv_sec, SDS.tkh, SDS.tk, SDS.shoc_ql, + SDS.shoc_cldfrac, SDS.pblh, SDS.shoc_mix, SDS.isotropy, SDS.w_sec, SDS.thl_sec, + SDS.qw_sec, SDS.qwthl_sec, SDS.wthl_sec, SDS.wqw_sec, SDS.wtke_sec, SDS.uw_sec, + SDS.vw_sec, SDS.w3, SDS.wqls_sec, SDS.brunt, SDS.shoc_ql2); + SDS.transpose(); // go back to C layout // Make sure output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp index 40303c6d0487..4d6b5dfdce1c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_mix_length_tests.cpp @@ -90,8 +90,14 @@ struct UnitWrap::UnitTest::TestCompShocMixLength { } } - // Call the fortran implementation - compute_shoc_mix_shoc_length(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + compute_shoc_mix_shoc_length_f(SDS.nlev, SDS.shcol, + SDS.tke, SDS.brunt, + SDS.zt_grid, + SDS.l_inf, SDS.shoc_mix); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp index 7ee84767a938..2d5f01c8ff27 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_check_pblh_tests.cpp @@ -67,8 +67,10 @@ struct UnitWrap::UnitTest::TestPblintdCheckPblh { } } - // Call the fortran implementation - pblintd_check_pblh(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_check_pblh_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.nlev, SDS.z, SDS.ustar, SDS.check, SDS.pblh); + SDS.transpose(); // go back to C layout // Check the result // Check that PBL height is greater than zero. This is an diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp index 41b61778630c..df65050acacc 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_cldcheck_tests.cpp @@ -79,8 +79,9 @@ struct UnitWrap::UnitTest::TestPblintdCldCheck { } } - // Call the fortran implementation - pblintd_cldcheck(SDS); + // Call the C++ implementation + SDS.transpose(); + shoc_pblintd_cldcheck_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.zi, SDS.cldn, SDS.pblh); // Check the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp index ca30fc4db5b0..9c33c7de68be 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_height_tests.cpp @@ -85,8 +85,11 @@ struct UnitWrap::UnitTest::TestPblintdHeight { REQUIRE(SDS.ustar[s+1] > SDS.ustar[s]); } - // Call the fortran implementation - pblintd_height(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, + SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); + SDS.transpose(); // go back to C layout // Check the result for(Int s = 0; s < shcol; ++s) { @@ -119,8 +122,11 @@ struct UnitWrap::UnitTest::TestPblintdHeight { SDS.thv[offset] = SDS.thv[offset-1] + 2; } - // Call the fortran implementation - pblintd_height(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, + SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); + SDS.transpose(); // go back to C layout // Check the result for(Int s = 0; s < shcol; ++s) { @@ -158,8 +164,11 @@ struct UnitWrap::UnitTest::TestPblintdHeight { } } - // Call the fortran implementation - pblintd_height(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_height_f(SDS.shcol, SDS.nlev, SDS.npbl, SDS.z, SDS.u, SDS.v, SDS.ustar, + SDS.thv, SDS.thv_ref, SDS.pblh, SDS.rino, SDS.check); + SDS.transpose(); // go back to C layout // Check that PBLH is zero (not modified) everywhere for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp index 910f71836ea7..b01c39eed95a 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_init_pot_test.cpp @@ -76,8 +76,8 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } } - // call the fortran implementation - pblintd_init_pot(SDS); + // call the C++ implementation + shoc_pblintd_init_pot_f(SDS.shcol, SDS.nlev, SDS.thl, SDS.ql, SDS.q, SDS.thv); // Check the result. // Verify that virtual potential temperature is idential @@ -125,8 +125,8 @@ struct UnitWrap::UnitTest::TestPblintdInitPot { } } - // Call the fortran implementation - pblintd_init_pot(SDS); + // Call the C++ implementation + shoc_pblintd_init_pot_f(SDS.shcol, SDS.nlev, SDS.thl, SDS.ql, SDS.q, SDS.thv); // Check test // Verify that column with condensate loading diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp index 55e11953db02..ccf3d110fc8d 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_surf_temp_tests.cpp @@ -83,8 +83,11 @@ struct UnitWrap::UnitTest::TestPblintdSurfTemp { } } - // Call the fortran implementation - pblintd_surf_temp(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_surf_temp_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.z, SDS.ustar, SDS.obklen, + SDS.kbfs, SDS.thv, SDS.tlv, SDS.pblh, SDS.check, SDS.rino); + SDS.transpose(); // go back to C layout // Check the result for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp index 7e27f8cce296..b5cd117228bf 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_pblintd_tests.cpp @@ -121,8 +121,12 @@ struct UnitWrap::UnitTest::TestPblintd { } } - // Call the fortran implementation - pblintd(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + pblintd_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.npbl, SDS.z, SDS.zi, + SDS.thl, SDS.ql, SDS.q, SDS.u, SDS.v, SDS.ustar, SDS.obklen, + SDS.kbfs, SDS.cldn, SDS.pblh); + SDS.transpose(); // go back to C layout // Make sure PBL height is reasonable // Should be larger than second lowest zi level and lower diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp index 1434cb27f4ec..ea2c95470d89 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_adv_sgs_tke_tests.cpp @@ -94,8 +94,10 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } } - // Call the fortran implementation - adv_sgs_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + adv_sgs_tke_f(SDS.nlev, SDS.shcol, SDS.dtime, SDS.shoc_mix, SDS.wthv_sec, SDS.sterm_zt, SDS.tk, SDS.tke, SDS.a_diss); + SDS.transpose(); // go back to C layout // Check to make sure that there has been // TKE growth @@ -159,8 +161,10 @@ struct UnitWrap::UnitTest::TestShocAdvSgsTke { } } - // Call the fortran implementation - adv_sgs_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + adv_sgs_tke_f(SDS.nlev, SDS.shcol, SDS.dtime, SDS.shoc_mix, SDS.wthv_sec, SDS.sterm_zt, SDS.tk, SDS.tke, SDS.a_diss); + SDS.transpose(); // go back to C layout // Check to make sure that the column with // the smallest length scale has larger diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp index 7cbb00255a76..eb68bbc11df1 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_column_stab_tests.cpp @@ -75,8 +75,10 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } } - // Call the fortran implementation - integ_column_stability(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + integ_column_stability_f(SDS.nlev, SDS.shcol, SDS.dz_zt, SDS.pres, SDS.brunt, SDS.brunt_int); + SDS.transpose(); // go back to C layout // Check test // Verify that output is zero @@ -107,8 +109,10 @@ struct UnitWrap::UnitTest::TestShocIntColStab { } } - // Call the fortran implementation - integ_column_stability(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + integ_column_stability_f(SDS.nlev, SDS.shcol, SDS.dz_zt, SDS.pres, SDS.brunt, SDS.brunt_int); + SDS.transpose(); // go back to C layout // Check test // Verify that output is negative diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp index 0baeb12b029a..c90128857cbb 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_isotropic_ts_tests.cpp @@ -81,8 +81,10 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } } - // Call the fortran implementation - isotropic_ts(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + isotropic_ts_f(SDS.nlev, SDS.shcol, SDS.brunt_int, SDS.tke, SDS.a_diss, SDS.brunt, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check that output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { @@ -145,8 +147,10 @@ struct UnitWrap::UnitTest::TestShocIsotropicTs { } } - // Call the fortran implementation - isotropic_ts(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + isotropic_ts_f(SDS.nlev, SDS.shcol, SDS.brunt_int, SDS.tke, SDS.a_diss, SDS.brunt, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check that output falls within reasonable bounds for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp index 8b088852af9c..0672ce3b3ae6 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_shr_prod_tests.cpp @@ -93,8 +93,10 @@ struct UnitWrap::UnitTest::TestShocShearProd { } } - // Call the fortran implementation - compute_shr_prod(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shr_prod_f(SDS.nlevi, SDS.nlev, SDS.shcol, SDS.dz_zi, SDS.u_wind, SDS.v_wind, SDS.sterm); + SDS.transpose(); // go back to C layout // Check test for(Int s = 0; s < shcol; ++s) { @@ -142,8 +144,10 @@ struct UnitWrap::UnitTest::TestShocShearProd { } } - // Call the fortran implementation - compute_shr_prod(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + compute_shr_prod_f(SDS.nlevi, SDS.nlev, SDS.shcol, SDS.dz_zi, SDS.u_wind, SDS.v_wind, SDS.sterm); + SDS.transpose(); // go back to C layout // Check test // Verify that shear term is zero everywhere diff --git a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp index bc975c9b59c1..f09ecc79c7d3 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_tke_tests.cpp @@ -153,8 +153,12 @@ struct UnitWrap::UnitTest::TestShocTke { } } - // Call the fortran implementation - shoc_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + shoc_tke_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.wthv_sec, SDS.shoc_mix, SDS.dz_zi, SDS.dz_zt, + SDS.pres, SDS.tabs, SDS.u_wind, SDS.v_wind, SDS.brunt, SDS.zt_grid, SDS.zi_grid, SDS.pblh, + SDS.tke, SDS.tk, SDS.tkh, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check test // Make sure that TKE has increased everwhere relative @@ -226,8 +230,12 @@ struct UnitWrap::UnitTest::TestShocTke { } } - // Call the fortran implementation - shoc_tke(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + shoc_tke_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.dtime, SDS.wthv_sec, SDS.shoc_mix, SDS.dz_zi, SDS.dz_zt, + SDS.pres, SDS.tabs, SDS.u_wind, SDS.v_wind, SDS.brunt, SDS.zt_grid, SDS.zi_grid, SDS.pblh, + SDS.tke, SDS.tk, SDS.tkh, SDS.isotropy); + SDS.transpose(); // go back to C layout // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp index 45268023faef..83cfb8d5707e 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_update_prognostics_implicit_tests.cpp @@ -258,10 +258,18 @@ struct UnitWrap::UnitTest::TestUpdatePrognosticsImplicit { } } - // Call the fortran implementation - update_prognostics_implicit(SDS); + // Call the C++ implementation + SDS.transpose(); // _f expects data in fortran layout + update_prognostics_implicit_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.num_tracer, SDS.dtime, + SDS.dz_zt, SDS.dz_zi, SDS.rho_zt, SDS.zt_grid, SDS.zi_grid, + SDS.tk, SDS.tkh, SDS.uw_sfc, SDS.vw_sfc, SDS.wthl_sfc, SDS.wqw_sfc, + SDS.wtracer_sfc, SDS.thetal, SDS.qw, SDS.tracer, SDS.tke, SDS.u_wind, SDS.v_wind); + SDS.transpose(); // go back to C layout + // Call linear interp to get rho value at surface for checking - linear_interp(SDSL); + SDSL.transpose(); // _f expects data in fortran layout + linear_interp_f(SDSL.x1, SDSL.x2, SDSL.y1, SDSL.y2, SDSL.km1, SDSL.km2, SDSL.ncol, SDSL.minthresh); + SDSL.transpose(); // go back to C layout // Check the result diff --git a/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp index 74990b2c43f2..4a743009f244 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_varorcovar_tests.cpp @@ -115,8 +115,14 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } } - // Call the fortran implementation for variance - calc_shoc_varorcovar(SDS); + // Call the C++ implementation for variance + SDS.transpose(); + // expects data in fortran layout + calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, + SDS.tunefac, SDS.isotropy_zi, + SDS.tkh_zi, SDS.dz_zi, + SDS.invar1, SDS.invar2, SDS.varorcovar); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -172,8 +178,14 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } } - // Call the fortran implementation for covariance - calc_shoc_varorcovar(SDS); + // Call the C++ implementation for covariance + SDS.transpose(); + // expects data in fortran layout + calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, + SDS.tunefac, SDS.isotropy_zi, + SDS.tkh_zi, SDS.dz_zi, + SDS.invar1, SDS.invar2, SDS.varorcovar); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { @@ -250,8 +262,14 @@ struct UnitWrap::UnitTest::TestShocVarorCovar { } } - // Call the fortran implementation for variance - calc_shoc_varorcovar(SDS); + // Call the C++ implementation for variance + SDS.transpose(); + // expects data in fortran layout + calc_shoc_varorcovar_f(SDS.shcol, SDS.nlev, SDS.nlevi, + SDS.tunefac, SDS.isotropy_zi, + SDS.tkh_zi, SDS.dz_zi, + SDS.invar1, SDS.invar2, SDS.varorcovar); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp b/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp index e8b1b5a8cdcd..7ac7c7c53160 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_vertflux_tests.cpp @@ -95,8 +95,11 @@ struct UnitWrap::UnitTest::TestCalcShocVertflux { } } - // Call the fortran implementation - calc_shoc_vertflux(SDS); + // Call the C++ implementation + SDS.transpose(); + // expects data in fortran layout + calc_shoc_vertflux_f(SDS.shcol, SDS.nlev, SDS.nlevi, SDS.tkh_zi, SDS.dz_zi, SDS.invar, SDS.vertflux); + SDS.transpose(); // Check the results for(Int s = 0; s < shcol; ++s) { diff --git a/components/eamxx/src/python/CMakeLists.txt b/components/eamxx/src/python/CMakeLists.txt index 8c7cf2aaba9c..d19bc05c240e 100644 --- a/components/eamxx/src/python/CMakeLists.txt +++ b/components/eamxx/src/python/CMakeLists.txt @@ -1,5 +1 @@ -find_package(pybind11 REQUIRED) -find_package(mpi4py REQUIRED) - -pybind11_add_module(pyeamxx pyeamxx.cpp) -target_link_libraries(pyeamxx PUBLIC mpi4py scream_share scream_io diagnostics eamxx_physics) +add_subdirectory(libpyscream) diff --git a/components/eamxx/src/python/libpyscream/CMakeLists.txt b/components/eamxx/src/python/libpyscream/CMakeLists.txt new file mode 100644 index 000000000000..7569d1cf7e60 --- /dev/null +++ b/components/eamxx/src/python/libpyscream/CMakeLists.txt @@ -0,0 +1,17 @@ +# Detect the installed nanobind package and import it into CMake +find_package(Python COMPONENTS Interpreter Development REQUIRED) +execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -m nanobind --cmake_dir + OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_ROOT) +find_package(nanobind REQUIRED) +find_package(mpi4py REQUIRED) + +nanobind_add_module(pyscream_ext pyscream_ext.cpp) +target_link_libraries(pyscream_ext PUBLIC + mpi4py + scream_share + scream_io + diagnostics + eamxx_physics + scream_test_support +) diff --git a/components/eamxx/src/python/pyatmproc.hpp b/components/eamxx/src/python/libpyscream/pyatmproc.hpp similarity index 60% rename from components/eamxx/src/python/pyatmproc.hpp rename to components/eamxx/src/python/libpyscream/pyatmproc.hpp index 4e7a88b21e67..219837c5f736 100644 --- a/components/eamxx/src/python/pyatmproc.hpp +++ b/components/eamxx/src/python/libpyscream/pyatmproc.hpp @@ -1,8 +1,6 @@ #ifndef PYATMPROC_HPP #define PYATMPROC_HPP -#include "physics/register_physics.hpp" -#include "diagnostics/register_diagnostics.hpp" #include "share/atm_process/atmosphere_process.hpp" #include "share/io/scorpio_input.hpp" #include "share/io/scream_output_manager.hpp" @@ -10,17 +8,20 @@ #include "pygrid.hpp" #include "pyfield.hpp" #include "pyparamlist.hpp" +#include "pyscream_ext.hpp" #include -#include -#include +#include +#include +#include + +namespace nb = nanobind; namespace scream { struct PyAtmProc { std::shared_ptr ap; - PyGrid phys_grid; std::map fields; util::TimeStamp t0; util::TimeStamp time; @@ -28,17 +29,14 @@ struct PyAtmProc { std::shared_ptr output_mgr; - PyAtmProc (const PyParamList& params, const PyGrid& phys_grid_in) - : phys_grid(phys_grid_in) + PyAtmProc (const nb::dict& d, const std::string& name) { - // Get the comm - const auto& comm = phys_grid.grid->get_comm(); + PyParamList params(d,name); - // Create a grids manager on the fly - auto gm = std::make_shared(phys_grid.grid); + // Get the comm + const auto& comm = PySession::get().comm; // Create the atm proc - register_physics(); auto& apf = AtmosphereProcessFactory::instance(); const auto& ap_type = params.pl.isParameter("Type") ? params.pl.get("Type") @@ -46,6 +44,7 @@ struct PyAtmProc { ap = apf.create(ap_type,comm,params.pl); // Create the fields + auto gm = PySession::get().gm; ap->set_grids(gm); create_fields(); } @@ -53,6 +52,11 @@ struct PyAtmProc { // I don't think virtual is needed, but just in case virtual ~PyAtmProc () = default; + PyParamList get_params () const { + PyParamList pypl(ap->get_params()); + return pypl; + } + void create_fields () { // Create fields that are input/output to the atm proc for (const auto& req : ap->get_required_field_requests()) { @@ -101,7 +105,7 @@ struct PyAtmProc { ap->initialize(t0,RunType::Initial); } - pybind11::list read_ic (const std::string& ic_filename) { + std::vector read_ic (const std::string& ic_filename) { // Get input fields, and read them from file (if present). // If field is not in the IC, user is responsible for setting // it to an initial value @@ -118,33 +122,68 @@ struct PyAtmProc { } } } - AtmosphereInput reader (ic_filename,phys_grid.grid,ic_fields,true); - reader.read_variables(); + if (ic_fields.size()>0) { + const auto& gn = ic_fields[0].get_header().get_identifier().get_grid_name(); + auto gm = PySession::get().gm; + auto grid = gm->get_grid(gn); + AtmosphereInput reader (ic_filename,grid,ic_fields,true); + reader.read_variables(); + } scorpio::release_file(ic_filename); - return pybind11::cast(missing); + return missing; + } + + std::vector list_fields(std::string ftype) { + std::vector fields_list; + for (const auto& field_pair : fields) { + const auto& field_identifier = field_pair.second.f.get_header().get_identifier(); + + if (ftype == "required" && ap->has_required_field(field_identifier)) { + fields_list.push_back(field_pair.first); + } else if (ftype == "computed" && ap->has_computed_field(field_identifier)) { + fields_list.push_back(field_pair.first); + } else if (ftype == "all") { + fields_list.push_back(field_pair.first); + } + } + return fields_list; + } + + std::vector list_all_fields() { + return list_fields("all"); + } + + std::vector list_required_fields() { + return list_fields("required"); + } + + std::vector list_computed_fields() { + return list_fields("computed"); } void setup_output (const std::string& yaml_file) { + auto comm = PySession::get().comm; // Load output params auto params = ekat::parse_yaml_file(yaml_file); // Stuff all fields in a field manager - auto fm = std::make_shared(phys_grid.grid); - fm->registration_begins(); - fm->registration_ends(); + auto gm = PySession::get().gm; + std::map> fms; + for (auto it : gm->get_repo()) { + fms[it.first] = std::make_shared(it.second); + fms[it.first]->registration_begins(); + fms[it.first]->registration_ends(); + } for (auto it : fields) { - fm->add_field(it.second.f); + const auto& gn = it.second.f.get_header().get_identifier().get_grid_name(); + fms.at(gn)->add_field(it.second.f); } - // Create a grids manager on the fly - auto gm = std::make_shared(phys_grid.grid); - - // Make sure diagnostics are available, then create the output mgr - register_diagnostics(); + // Create/setup the output mgr output_mgr = std::make_shared(); - output_mgr->setup(phys_grid.grid->get_comm(),params,fm,gm,t0,t0,false); + output_mgr->setup(comm,params,fms,gm,t0,t0,false); output_mgr->set_logger(ap->get_logger()); } @@ -158,15 +197,19 @@ struct PyAtmProc { }; // Register type in the py module -inline void pybind_pyatmproc(pybind11::module& m) +inline void nb_pyatmproc(nb::module_& m) { - pybind11::class_(m,"AtmProc") - .def(pybind11::init()) + nb::class_(m,"AtmProc") + .def(nb::init()) .def("get_field",&PyAtmProc::get_field) .def("initialize",&PyAtmProc::initialize) + .def("get_params",&PyAtmProc::get_params) .def("setup_output",&PyAtmProc::setup_output) .def("run",&PyAtmProc::run) - .def("read_ic",&PyAtmProc::read_ic); + .def("read_ic",&PyAtmProc::read_ic) + .def("list_all_fields",&PyAtmProc::list_all_fields) + .def("list_required_fields",&PyAtmProc::list_required_fields) + .def("list_computed_fields",&PyAtmProc::list_computed_fields); } } // namespace scream diff --git a/components/eamxx/src/python/pyfield.hpp b/components/eamxx/src/python/libpyscream/pyfield.hpp similarity index 82% rename from components/eamxx/src/python/pyfield.hpp rename to components/eamxx/src/python/libpyscream/pyfield.hpp index eeb4125cecc2..96dae4b3e454 100644 --- a/components/eamxx/src/python/pyfield.hpp +++ b/components/eamxx/src/python/libpyscream/pyfield.hpp @@ -4,9 +4,11 @@ #include "share/field/field.hpp" #include "share/field/field_utils.hpp" -#include -#include -#include +#include +#include +#include + +namespace nb = nanobind; namespace scream { @@ -24,7 +26,8 @@ struct PyField { f.allocate_view(); } - pybind11::array get () const { + template + nb::ndarray get () const { const auto& fh = f.get_header(); const auto& fid = fh.get_identifier(); @@ -39,10 +42,14 @@ struct PyField { // NOTE: since the field may be padded, the strides do not necessarily // match the dims. Also, the strides must be grabbed from the // actual view, since the layout doesn't know them. - pybind11::array::ShapeContainer shape (fid.get_layout().dims()); + int shape_t = f.rank(); + size_t shape[shape_t] = {0}; + for (int i=0; i strides; - pybind11::dtype dt; + nb::dlpack::dtype dt; switch (fid.data_type()) { case DataType::IntType: dt = get_dt_and_set_strides(strides); @@ -59,8 +66,8 @@ struct PyField { // NOTE: you MUST set the parent handle, or else you won't have view semantic auto data = f.get_internal_view_data_unsafe(); - auto this_obj = pybind11::cast(this); - return pybind11::array(dt,shape,strides,data,pybind11::handle(this_obj)); + auto this_obj = nb::cast(this); + return nb::ndarray(data, shape_t, shape, nb::handle(this_obj), strides.data(), dt); } void sync_to_host () { @@ -75,7 +82,7 @@ struct PyField { private: template - pybind11::dtype get_dt_and_set_strides (std::vector& strides) const + nb::dlpack::dtype get_dt_and_set_strides (std::vector& strides) const { strides.resize(f.rank()); switch (f.rank()) { @@ -126,15 +133,15 @@ struct PyField { " - field rnak: " + std::to_string(f.rank()) + "\n"); } - return pybind11::dtype::of(); + return nb::dtype(); } }; -inline void pybind_pyfield (pybind11::module& m) { +inline void nb_pyfield (nb::module_& m) { // Field class - pybind11::class_(m,"Field") - .def(pybind11::init<>()) - .def("get",&PyField::get) + nb::class_(m,"Field") + .def(nb::init<>()) + .def("get",&PyField::get) .def("sync_to_host",&PyField::sync_to_host) .def("sync_to_dev",&PyField::sync_to_dev) .def("print",&PyField::print); diff --git a/components/eamxx/src/python/libpyscream/pygrid.hpp b/components/eamxx/src/python/libpyscream/pygrid.hpp new file mode 100644 index 000000000000..83cb78d6729e --- /dev/null +++ b/components/eamxx/src/python/libpyscream/pygrid.hpp @@ -0,0 +1,51 @@ +#ifndef PYGRID_HPP +#define PYGRID_HPP + +#include "share/grid/mesh_free_grids_manager.hpp" + +#include "pyscream_ext.hpp" + +#include +#include + +#include + +namespace nb = nanobind; + +namespace scream { + +inline void create_grids_manager (int ncols, int nlevs, const std::string& latlon_nc_file) +{ + EKAT_REQUIRE_MSG (PySession::get().inited, + "Error! You did not initialize pyscream, or you already finalized it!\n"); + auto& comm = PySession::get().comm; + ekat::ParameterList gm_params; + std::vector grids_names = {"Physics"}; + auto& pl = gm_params.sublist("Physics"); + pl.set("type",std::string("point_grid")); + pl.set("number_of_global_columns",ncols); + pl.set("number_of_vertical_levels",nlevs); + gm_params.set("grids_names",grids_names); + gm_params.set("geo_data_source",std::string("CREATE_EMPTY_DATA")); + + if (latlon_nc_file!="") { + gm_params.set("geo_data_source",std::string("IC_FILE")); + gm_params.set("ic_filename",latlon_nc_file); + } + + PySession::get().gm = create_mesh_free_grids_manager (comm, gm_params); + PySession::get().gm->build_grids(); +} +inline void create_grids_manager (int ncols, int nlevs) +{ + create_grids_manager(ncols,nlevs,""); +} + +inline void nb_pygrid (nb::module_& m) { + m.def("create_grids_manager",nb::overload_cast(&create_grids_manager)); + m.def("create_grids_manager",nb::overload_cast(&create_grids_manager)); +} + +} // namespace scream + +#endif // PYGRID_HPP diff --git a/components/eamxx/src/python/libpyscream/pyparamlist.hpp b/components/eamxx/src/python/libpyscream/pyparamlist.hpp new file mode 100644 index 000000000000..8da57a24bd35 --- /dev/null +++ b/components/eamxx/src/python/libpyscream/pyparamlist.hpp @@ -0,0 +1,159 @@ +#ifndef PYPARAMLIST_HPP +#define PYPARAMLIST_HPP + +#include + +#include +#include +#include +#include + +#include + +namespace nb = nanobind; + +namespace scream { + +struct PyParamList { + ekat::ParameterList pl; + std::reference_wrapper pl_ref; + + PyParamList (ekat::ParameterList& src) + : pl_ref(src) + {} + + PyParamList(const nb::dict& d) + : PyParamList(d,"") + {} + + PyParamList(const nb::dict& d, const std::string& name) + : pl(name) + , pl_ref(pl) + { + parse_dict(d,pl); + } + + PyParamList sublist (const std::string& name) { + PyParamList spl(pl.sublist(name)); + return spl; + } + + bool get_bool (const std::string& name) const { + return pl_ref.get().get(name); + } + int get_int (const std::string& name) const { + return pl_ref.get().get(name); + } + double get_dbl (const std::string& name) const { + return pl_ref.get().get(name); + } + std::string get_str (const std::string& name) const { + return pl_ref.get().get(name); + } + + std::vector get_int_vec (const std::string& name) const { + return pl_ref.get().get>(name); + } + std::vector get_dbl_vec (const std::string& name) const { + return pl_ref.get().get>(name); + } + std::vector get_str_vec (const std::string& name) const { + return pl_ref.get().get>(name); + } + + template + void set (const std::string& name, T val) { + pl_ref.get().set(name,val); + } + + void print () { + pl_ref.get().print(); + } + +private: + + void parse_dict(const nb::dict& d, ekat::ParameterList& p) { + for (auto item : d) { + auto key = nb::cast(item.first); + if (nb::isinstance(item.second)) { + auto pystr = nb::str(item.second); + p.set(key,nb::cast(pystr)); + } else if (nb::isinstance(item.second)) { + auto pyint = nb::cast(item.second); + p.set(key,nb::cast(pyint)); + } else if (nb::isinstance(item.second)) { + auto pyint = nb::cast(item.second); + p.set(key,nb::cast(pyint)); + } else if (nb::isinstance(item.second)) { + auto pydouble = nb::cast(item.second); + p.set(key,nb::cast(pydouble)); + } else if (nb::isinstance(item.second)) { + auto pylist = nb::cast(item.second); + parse_list(pylist,p,key); + } else if (nb::isinstance(item.second)) { + auto pydict = nb::cast(item.second); + parse_dict(pydict,p.sublist(key)); + } else { + EKAT_ERROR_MSG ("Unsupported/unrecognized dict entry type.\n"); + } + } + } + + void parse_list (const nb::list& l, ekat::ParameterList&p, const std::string& key) { + EKAT_REQUIRE_MSG (nb::len(l)>0, + "Error! Cannot deduce type for dictionary list entry '" + key + "'\n"); + auto first = l[0]; + bool are_ints = nb::isinstance(first); + bool are_floats = nb::isinstance(first); + bool are_strings = nb::isinstance(first); + if (are_ints) { + parse_list_impl(l,p,key); + } else if (are_floats) { + parse_list_impl(l,p,key); + } else if (are_strings) { + parse_list_impl(l,p,key); + } else { + EKAT_ERROR_MSG ("Unrecognized/unsupported list entry type.\n"); + } + } + + template + void parse_list_impl(const nb::list& l, ekat::ParameterList& p, const std::string& key) { + std::vector vals; + for (auto item : l) { + EKAT_REQUIRE_MSG (nb::isinstance(item), + "Error! Inconsistent types in list entries.\n"); + auto item_py = nb::cast(item); + vals.push_back(nb::cast(item_py)); + } + p.set(key,vals); + } +}; + +inline void nb_pyparamlist (nb::module_& m) +{ + // Param list + nb::class_(m,"ParameterList") + .def(nb::init()) + .def(nb::init()) + .def("sublist",&PyParamList::sublist) + .def("print",&PyParamList::print) + .def("set",&PyParamList::set) + .def("set",&PyParamList::set) + .def("set",&PyParamList::set) + .def("set",&PyParamList::set) + .def("set",&PyParamList::set>) + .def("set",&PyParamList::set>) + .def("set",&PyParamList::set>) + .def("get_bool",&PyParamList::get_bool) + .def("get_int",&PyParamList::get_int) + .def("get_dbl",&PyParamList::get_dbl) + .def("get_str",&PyParamList::get_str) + .def("get_int_vec",&PyParamList::get_int_vec) + .def("get_dbl_vec",&PyParamList::get_dbl_vec) + .def("get_str_vec",&PyParamList::get_str_vec); +} + +} // namespace scream + +#endif // PYPARAMLIST_HPP diff --git a/components/eamxx/src/python/pyeamxx.cpp b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp similarity index 57% rename from components/eamxx/src/python/pyeamxx.cpp rename to components/eamxx/src/python/libpyscream/pyscream_ext.cpp index 5834ca10f74f..892d40e8ca3c 100644 --- a/components/eamxx/src/python/pyeamxx.cpp +++ b/components/eamxx/src/python/libpyscream/pyscream_ext.cpp @@ -7,11 +7,11 @@ #include -#include +#include #include -namespace py = pybind11; +namespace nb = nanobind; namespace scream { @@ -19,33 +19,46 @@ void initialize (MPI_Comm mpi_comm) { ekat::Comm comm(mpi_comm); initialize_scream_session(comm.am_i_root()); scorpio::init_subsystem(comm); + + // Register everything in the eamxx factories + register_physics(); + register_dynamics(); + register_diagnostics(); + + auto& s = PySession::get(); + s.comm = comm; + s.inited = true; } void initialize () { initialize(MPI_COMM_WORLD); } -void initialize (pybind11::object py_comm) { +void initialize (nb::object py_comm) { initialize(get_c_comm(py_comm)); } void finalize () { + auto& s = PySession::get(); + s.gm = nullptr; + s.inited = false; + scorpio::finalize_subsystem(); finalize_scream_session(); } -PYBIND11_MODULE (pyeamxx,m) { +NB_MODULE (pyscream_ext,m) { m.doc() = "Python interfaces to certain EAMxx infrastructure code"; // Scream Session - m.def("init",py::overload_cast<>(&initialize)); - m.def("init",py::overload_cast(&initialize)); + m.def("init",nb::overload_cast<>(&initialize)); + m.def("init",nb::overload_cast(&initialize)); m.def("finalize",&finalize); // Call all other headers' registration routines - pybind_pyparamlist(m); - pybind_pyfield(m); - pybind_pygrid(m); - pybind_pyatmproc(m); + nb_pyparamlist(m); + nb_pyfield(m); + nb_pygrid(m); + nb_pyatmproc(m); } } // namespace scream diff --git a/components/eamxx/src/python/libpyscream/pyscream_ext.hpp b/components/eamxx/src/python/libpyscream/pyscream_ext.hpp new file mode 100644 index 000000000000..781c195919dd --- /dev/null +++ b/components/eamxx/src/python/libpyscream/pyscream_ext.hpp @@ -0,0 +1,30 @@ +#ifndef PYSCREAM_HPP +#define PYSCREAM_HPP + +#include "physics/register_physics.hpp" +#include "diagnostics/register_diagnostics.hpp" +#include "dynamics/register_dynamics.hpp" + +#include "share/grid/grids_manager.hpp" + +#include + +namespace scream { + +struct PySession { + static PySession& get () { + static PySession s; + return s; + } + + ekat::Comm comm; + std::shared_ptr gm; + bool inited = false; +private: + + PySession () = default; +}; + +} // namespace scream + +#endif // PYSCREAM_HPP diff --git a/components/eamxx/src/python/pyutils.hpp b/components/eamxx/src/python/libpyscream/pyutils.hpp similarity index 74% rename from components/eamxx/src/python/pyutils.hpp rename to components/eamxx/src/python/libpyscream/pyutils.hpp index 6467df57eda1..b02ec9a36a12 100644 --- a/components/eamxx/src/python/pyutils.hpp +++ b/components/eamxx/src/python/libpyscream/pyutils.hpp @@ -1,15 +1,15 @@ #ifndef PYUTILS_HPP #define PYUTILS_HPP -#include +#include #include #include #include -MPI_Comm get_c_comm (pybind11::object py_comm) { +MPI_Comm get_c_comm (nb::object py_comm) { if (import_mpi4py() < 0) { - throw pybind11::error_already_set(); + throw nb::python_error(); } auto py_src = py_comm.ptr(); if (not PyObject_TypeCheck(py_src, &PyMPIComm_Type)) { diff --git a/components/eamxx/src/python/pyeamxx/__init__.py b/components/eamxx/src/python/pyeamxx/__init__.py deleted file mode 100644 index 75a36e148291..000000000000 --- a/components/eamxx/src/python/pyeamxx/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -""" - This file will serve as a way to organize and expose - libpyeamxx internals to the rest of pyeamxx -""" - -from libpyeamxx.libpyeamxx_ext import AtmProc -from libpyeamxx.libpyeamxx_ext import Grid -from libpyeamxx.libpyeamxx_ext import ParameterList -from libpyeamxx.libpyeamxx_ext import init -from libpyeamxx.libpyeamxx_ext import Field -from libpyeamxx.libpyeamxx_ext import P3 -from libpyeamxx.libpyeamxx_ext import finalize - - -__all__ = [ - 'init', - 'finalize', - 'AtmProc', - 'Grid', - 'ParameterList', - 'Field', - 'P3', -] diff --git a/components/eamxx/src/python/pygrid.hpp b/components/eamxx/src/python/pygrid.hpp deleted file mode 100644 index 874b1aef17a0..000000000000 --- a/components/eamxx/src/python/pygrid.hpp +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef PYGRID_HPP -#define PYGRID_HPP - -#include "share/grid/grids_manager.hpp" -#include "share/grid/point_grid.hpp" -#include "pyfield.hpp" -#include "pyutils.hpp" - -#include - -#include - -namespace scream { - -// Small grids manager class, to hold a pre-built grid -// We will use this to build a GM on the fly from a single grid -class SingleGridGM : public GridsManager -{ -public: - SingleGridGM (const std::shared_ptr& grid) - { - add_grid(grid); - } - - std::string name () const override { return "SingleGridGM"; } - - void build_grids () override {} - -protected: - remapper_ptr_type - do_create_remapper (const grid_ptr_type /* from_grid */, - const grid_ptr_type /* to_grid */) const override - { - EKAT_ERROR_MSG ("Error! do_create_remapper not implemented for SingleGridGM.\n"); - } -}; - -struct PyGrid { - std::shared_ptr grid; - - PyGrid () = default; - - PyGrid(const std::string& name, int ncols, int nlevs) - : PyGrid (name,ncols,nlevs,MPI_COMM_WORLD) - {} - - PyGrid(const std::string& name, int ncols, int nlevs, pybind11::object py_comm) - : PyGrid (name,ncols,nlevs,get_c_comm(py_comm)) - {} - - PyGrid(const std::string& name, int ncols, int nlevs, MPI_Comm mpi_comm) - { - ekat::Comm comm(mpi_comm); - grid = create_point_grid(name,ncols,nlevs,comm); - } -}; - -inline void pybind_pygrid (pybind11::module& m) { - pybind11::class_(m,"Grid") - .def(pybind11::init()) - .def(pybind11::init()); -} - -} // namespace scream - -#endif // PYGRID_HPP diff --git a/components/eamxx/src/python/pyparamlist.hpp b/components/eamxx/src/python/pyparamlist.hpp deleted file mode 100644 index 00ee1c894f84..000000000000 --- a/components/eamxx/src/python/pyparamlist.hpp +++ /dev/null @@ -1,92 +0,0 @@ -#ifndef PYPARAMLIST_HPP -#define PYPARAMLIST_HPP - -#include - -#include -#include -#include - -namespace scream { - -struct PyParamList { - ekat::ParameterList pl; - - PyParamList(const pybind11::dict& d) { - parse_dict(d,pl); - } - - PyParamList(const pybind11::dict& d, const std::string& name) { - pl.rename(name); - parse_dict(d,pl); - } - - void parse_dict(const pybind11::dict& d, ekat::ParameterList& p) { - for (auto item : d) { - const std::string key = pybind11::str(item.first); - if (pybind11::isinstance(item.second)) { - auto pystr = pybind11::str(item.second); - p.set(key,pystr.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pyint = pybind11::cast(item.second); - p.set(key,pyint.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pyint = pybind11::cast(item.second); - p.set(key,pyint.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pydouble = pybind11::cast(item.second); - p.set(key,pydouble.cast()); - } else if (pybind11::isinstance(item.second)) { - auto pylist = pybind11::cast(item.second); - parse_list(pylist,p,key); - } else if (pybind11::isinstance(item.second)) { - auto pydict = pybind11::cast(item.second); - parse_dict(pydict,p.sublist(key)); - } else { - EKAT_ERROR_MSG ("Unsupported/unrecognized dict entry type.\n"); - } - } - } - - void parse_list (const pybind11::list& l, ekat::ParameterList&p, const std::string& key) { - EKAT_REQUIRE_MSG (pybind11::len(l)>0, - "Error! Cannot deduce type for dictionary list entry '" + key + "'\n"); - auto first = l[0]; - bool are_ints = pybind11::isinstance(first); - bool are_floats = pybind11::isinstance(first); - bool are_strings = pybind11::isinstance(first); - if (are_ints) { - parse_list_impl(l,p,key); - } else if (are_floats) { - parse_list_impl(l,p,key); - } else if (are_strings) { - parse_list_impl(l,p,key); - } else { - EKAT_ERROR_MSG ("Unrecognized/unsupported list entry type.\n"); - } - } - - template - void parse_list_impl(const pybind11::list& l, ekat::ParameterList& p, const std::string& key) { - std::vector vals; - for (auto item : l) { - EKAT_REQUIRE_MSG (pybind11::isinstance(item), - "Error! Inconsistent types in list entries.\n"); - auto item_py = pybind11::cast(item); - vals.push_back(item_py.template cast()); - } - p.set(key,vals); - } -}; - -inline void pybind_pyparamlist (pybind11::module& m) -{ - // Param list - pybind11::class_(m,"ParameterList") - .def(pybind11::init()) - .def(pybind11::init()); -} - -} // namespace scream - -#endif // PYPARAMLIST_HPP diff --git a/components/eamxx/src/python/pyproject.toml b/components/eamxx/src/python/pyproject.toml index fa5d6fc44817..490ea99c143e 100644 --- a/components/eamxx/src/python/pyproject.toml +++ b/components/eamxx/src/python/pyproject.toml @@ -3,15 +3,15 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] -name = "pyeamxx" +name = "pyscream" version = "0.0.2" dependencies = ["numpy", "mpi4py"] [tool.setuptools.packages.find] where = ["."] -include = ["pyeamxx", "libpyeamxx"] +include = ["pyscream", "libpyscream"] exclude = ["build_src", "tests"] namespaces = true [tool.setuptools.package-data] -"libpyeamxx" = ["*.so*"] +"libpyscream" = ["*.so*"] diff --git a/components/eamxx/src/python/pyscream/__init__.py b/components/eamxx/src/python/pyscream/__init__.py new file mode 100644 index 000000000000..e76f52170fb3 --- /dev/null +++ b/components/eamxx/src/python/pyscream/__init__.py @@ -0,0 +1,20 @@ +""" + This file will serve as a way to organize and expose + libpyscream internals to the rest of pyscream +""" + +from libpyscream.pyscream_ext import init +from libpyscream.pyscream_ext import finalize +from libpyscream.pyscream_ext import Field +from libpyscream.pyscream_ext import AtmProc +from libpyscream.pyscream_ext import ParameterList +from libpyscream.pyscream_ext import create_grids_manager + +__all__ = [ + "init", + "finalize", + "Field", + "AtmProc", + "ParameterList", + "create_grids_manager", +] diff --git a/components/eamxx/src/python/readme b/components/eamxx/src/python/readme index f5ed48f058ee..c16c32a5d217 100644 --- a/components/eamxx/src/python/readme +++ b/components/eamxx/src/python/readme @@ -1,7 +1,7 @@ INFO: - EAMxx python bindings -- pyeamxx is where we will house the python code -- libpyeamxx is where we will house the extensions +- pyscream is where we will house the python code +- libpyscream is where we will house the extensions - packaging moved to https://github.com/mahf708/experimental-scream-feedstock TODO: @@ -11,5 +11,5 @@ TODO: - decide archs/pythons/mpis to tgt USER: -- conda install pyeamxx -c mahf708/label/$mac (mac is chrysalis or pm-cpu) +- conda install pyscream -c mahf708/label/$mac (mac is chrysalis or pm-cpu) - see example in components/eamxx/tests/python/pyp3 diff --git a/components/eamxx/src/python/tests/basic_test.py b/components/eamxx/src/python/tests/basic_test.py deleted file mode 100644 index 26e43cad0d2b..000000000000 --- a/components/eamxx/src/python/tests/basic_test.py +++ /dev/null @@ -1,20 +0,0 @@ -# luca's verbatim example works from any dir now! -from pyeamxx import pyscream as ps -import numpy as np - -ps.init() - -v = np.zeros(shape=(10,2)) - -for i in range(0,10): - for j in range(0,2): - v[i,j] = i*2 + j + 1 - -print(f"v:{v}") - -f = ps.Field("T_mid",v) - -f.print() - -f.cleanup() -ps.finalize() diff --git a/components/eamxx/src/scream_config.h.in b/components/eamxx/src/scream_config.h.in index 77b1bf04ff0c..2bd8c72f013b 100644 --- a/components/eamxx/src/scream_config.h.in +++ b/components/eamxx/src/scream_config.h.in @@ -45,8 +45,12 @@ // Whether or not to run RRTMGP debug checks #cmakedefine SCREAM_RRTMGP_DEBUG -// Whether monolithic kernels are on +// Whether or not small kernels are used in ALL targets that support them #cmakedefine SCREAM_SMALL_KERNELS +// Whether or not small kernels are used in P3 +#cmakedefine SCREAM_P3_SMALL_KERNELS +// Whether or not small kernels are used in SHOC +#cmakedefine SCREAM_SHOC_SMALL_KERNELS // The sha of the last commit #define EAMXX_GIT_VERSION "${EAMXX_GIT_VERSION}" diff --git a/components/eamxx/src/share/atm_process/SCDataManager.hpp b/components/eamxx/src/share/atm_process/SCDataManager.hpp index 02af15423e62..76bc9e5bd843 100644 --- a/components/eamxx/src/share/atm_process/SCDataManager.hpp +++ b/components/eamxx/src/share/atm_process/SCDataManager.hpp @@ -22,7 +22,11 @@ struct SCDataManager { ~SCDataManager() = default; void setup_internals (const int num_cpl_fields, const int num_scream_fields, const int field_size, - Real* field_data_ptr, char* field_names, int* field_cpl_indices_ptr, + Real* field_data_ptr, +#ifdef HAVE_MOAB + Real* field_data_moab_ptr, +#endif + char* field_names, int* field_cpl_indices_ptr, int* field_vector_components_ptr, Real* field_constant_multiple_ptr, bool* transfer_during_init_ptr) { @@ -38,6 +42,9 @@ struct SCDataManager { EKAT_ASSERT_MSG(field_constant_multiple_ptr !=nullptr, "Error! Ptr for constant multiple is null."); EKAT_ASSERT_MSG(transfer_during_init_ptr !=nullptr, "Error! Ptr for initial transfer boolean is null."); m_field_data = decltype(m_field_data) (field_data_ptr, m_field_size, m_num_cpl_fields); +#ifdef HAVE_MOAB + m_field_data_moab = decltype(m_field_data_moab) (field_data_moab_ptr, m_num_cpl_fields, m_field_size); +#endif m_field_cpl_indices = decltype(m_field_cpl_indices) (field_cpl_indices_ptr, m_num_scream_fields); m_field_vector_components = decltype(m_field_vector_components) (field_vector_components_ptr, m_num_scream_fields); m_field_constant_multiple = decltype(m_field_constant_multiple) (field_constant_multiple_ptr, m_num_scream_fields); @@ -65,7 +72,11 @@ struct SCDataManager { Real* get_field_data_ptr () const { return m_field_data.data(); } - +#ifdef HAVE_MOAB + Real* get_field_data_moab_ptr () const { + return m_field_data_moab.data(); + } +#endif Real get_field_data_view_entry(const int i, const int f) { return m_field_data(i, f); } @@ -105,6 +116,9 @@ struct SCDataManager { int m_num_scream_fields; view_2d m_field_data; +#ifdef HAVE_MOAB + view_2d m_field_data_moab; +#endif name_t* m_field_names; view_1d m_field_cpl_indices; view_1d m_field_vector_components; diff --git a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp index 6b2f2fd080bb..533090e4e7cb 100644 --- a/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp +++ b/components/eamxx/src/share/atm_process/atmosphere_process_hash.cpp @@ -129,9 +129,9 @@ ::print_global_state_hash (const std::string& label, const bool in, const bool o if (m_comm.am_i_root()) for (int i = 0; i < nslot; ++i) if (show[i]) - fprintf(stderr, "exxhash> %4d-%9.5f %1d %16lld (%s)\n", + fprintf(stderr, "exxhash> %4d-%9.5f %1d %16" PRIx64 " (%s)\n", timestamp().get_year(), timestamp().frac_of_year_in_days(), - i, (long long int)gaccum[i], label.c_str()); + i, gaccum[i], label.c_str()); } void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) const { @@ -140,8 +140,8 @@ void AtmosphereProcess::print_fast_global_state_hash (const std::string& label) HashType gaccum; bfbhash::all_reduce_HashType(m_comm.mpi_comm(), &laccum, &gaccum, 1); if (m_comm.am_i_root()) - fprintf(stderr, "bfbhash> %14d %16lld (%s)\n", - timestamp().get_num_steps(), (long long int) gaccum, label.c_str()); + fprintf(stderr, "bfbhash> %14d %16" PRIx64 " (%s)\n", + timestamp().get_num_steps(), gaccum, label.c_str()); } } // namespace scream diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 320e10a60273..00a2c9b593fc 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -60,22 +60,62 @@ Field::clone(const std::string& name) const { } void Field:: -sync_to_host () const { +sync_to_host (const bool fence) const { // Sanity check EKAT_REQUIRE_MSG (is_allocated(), "Error! Input field must be allocated in order to sync host and device views.\n"); - Kokkos::deep_copy(m_data.h_view,m_data.d_view); + // Check for early return if Host and Device are the same memory space + if (host_and_device_share_memory_space()) return; + + // We allow sync_to_host for constant fields. Temporarily disable read only flag. + const bool original_read_only = m_is_read_only; + m_is_read_only = false; + + switch (data_type()) { + case DataType::IntType: + sync_views_impl(); + break; + case DataType::FloatType: + sync_views_impl(); + break; + case DataType::DoubleType: + sync_views_impl(); + break; + default: + EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_host.\n"); + } + + if (fence) Kokkos::fence(); + + // Return field to read-only state + m_is_read_only = original_read_only; } void Field:: -sync_to_dev () const { +sync_to_dev (const bool fence) const { // Sanity check EKAT_REQUIRE_MSG (is_allocated(), "Error! Input field must be allocated in order to sync host and device views.\n"); - // Ensure host view was created (lazy construction) - Kokkos::deep_copy(m_data.d_view,m_data.h_view); + // Check for early return if Host and Device are the same memory space + if (host_and_device_share_memory_space()) return; + + switch (data_type()) { + case DataType::IntType: + sync_views_impl(); + break; + case DataType::FloatType: + sync_views_impl(); + break; + case DataType::DoubleType: + sync_views_impl(); + break; + default: + EKAT_ERROR_MSG("Error! Unrecognized field data type in Field::sync_to_dev.\n"); + } + + if (fence) Kokkos::fence(); } Field Field:: @@ -92,7 +132,7 @@ subfield (const std::string& sf_name, const ekat::units::Units& sf_units, "Error! Subview dimension index must be either 0 or 1.\n"); // Create identifier for subfield - FieldIdentifier sf_id(sf_name,lt.clone().strip_dim(idim),sf_units,id.get_grid_name()); + FieldIdentifier sf_id(sf_name,lt.clone().strip_dim(idim),sf_units,id.get_grid_name(),id.data_type()); // Create empty subfield, then set header and views // Note: we can access protected members, since it's the same type @@ -101,6 +141,14 @@ subfield (const std::string& sf_name, const ekat::units::Units& sf_units, sf.m_data = m_data; sf.m_is_read_only = m_is_read_only; + if (not sf.m_header->get_alloc_properties().contiguous() and + not sf.host_and_device_share_memory_space()) { + // If subfield is not contiguous and Host and Device do not + // share a memory space, we must initialize the helper field + // for sync_to functions. + sf.initialize_contiguous_helper_field(); + } + return sf; } @@ -133,7 +181,7 @@ Field Field::subfield(const std::string& sf_name, auto sf_layout = lt.clone(); sf_layout.reset_dim(idim, index_end - index_beg); // Create identifier for subfield - FieldIdentifier sf_id(sf_name, sf_layout, sf_units, id.get_grid_name()); + FieldIdentifier sf_id(sf_name, sf_layout, sf_units, id.get_grid_name(), id.data_type()); // Create empty subfield, then set header and views // Note: we can access protected members, since it's the same type @@ -142,6 +190,14 @@ Field Field::subfield(const std::string& sf_name, index_end); sf.m_data = m_data; + if (not sf.m_header->get_alloc_properties().contiguous() and + not sf.host_and_device_share_memory_space()) { + // If subfield is not contiguous and Host and Device do not + // share a memory space, we must initialize the helper field + // for sync_to functions. + sf.initialize_contiguous_helper_field(); + } + return sf; } diff --git a/components/eamxx/src/share/field/field.hpp b/components/eamxx/src/share/field/field.hpp index e48a570d8131..a338199f2ef0 100644 --- a/components/eamxx/src/share/field/field.hpp +++ b/components/eamxx/src/share/field/field.hpp @@ -193,16 +193,19 @@ class Field { // Note: this class takes no responsibility in keeping track of whether // a sync is required in either direction. Mainly because we expect // host views to be seldom used, and even less frequently modified. - void sync_to_host () const; - void sync_to_dev () const; + // The fence input controls whether a fence is done at the end of the sync. + // If multiple syncs are performed in a row on different data, the user may + // want to run them asynchronously and fence the final sync_to call. + void sync_to_host (const bool fence = true) const; + void sync_to_dev (const bool fence = true) const; // Set the field to a constant value (on host or device) template - void deep_copy (const T value); + void deep_copy (const T value) const; // Copy the data from one field to this field template - void deep_copy (const Field& src); + void deep_copy (const Field& src) const; // Updates this field y as y=alpha*x+beta*y // NOTE: ST=void is just so we can give a default to HD, @@ -278,15 +281,46 @@ class Field { // Allocate the actual view void allocate_view (); + // Create contiguous helper field for running sync_to_host + // and sync_to_device with non-contiguous fields + void initialize_contiguous_helper_field () { + EKAT_REQUIRE_MSG(not m_header->get_alloc_properties().contiguous(), + "Error! We should not setup contiguous helper field " + "for an already contiguous field.\n"); + EKAT_REQUIRE_MSG(not host_and_device_share_memory_space(), + "Error! We should not setup contiguous helper field for a field " + "when host and device share a memory space.\n"); + + auto id = m_header->get_identifier(); + Field contig(id.alias(name()+std::string("_contiguous"))); + contig.allocate_view(); + + // Sanity check + EKAT_REQUIRE_MSG(contig.get_header().get_alloc_properties().contiguous(), + "Error! Contiguous helper field must be contiguous.\n"); + + m_contiguous_field = std::make_shared(contig); + } + + inline bool host_and_device_share_memory_space() const { + EKAT_REQUIRE_MSG(is_allocated(), + "Error! Must allocate view before querying " + "host_and_device_share_memory_space().\n"); + return m_data.h_view.data() == m_data.d_view.data(); + } + #ifndef KOKKOS_ENABLE_CUDA // Cuda requires methods enclosing __device__ lambda's to be public protected: #endif + template + void sync_views_impl () const; + template - void deep_copy_impl (const ST value); + void deep_copy_impl (const ST value) const; template - void deep_copy_impl (const Field& src); + void deep_copy_impl (const Field& src) const; template void update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val); @@ -337,8 +371,14 @@ class Field { // Actual data. dual_view_t m_data; - // Whether this field is read-only - bool m_is_read_only = false; + // Field needed for sync host/device in case of non-contiguous + // field when host and device do not share a memory space. + std::shared_ptr m_contiguous_field; + + // Whether this field is read-only. This is given + // mutable keyword since it needs to be turned off/on + // to allow sync_to_host for constant, read-only fields. + mutable bool m_is_read_only = false; }; // We use this to find a Field in a std container. diff --git a/components/eamxx/src/share/field/field_impl.hpp b/components/eamxx/src/share/field/field_impl.hpp index 67753605864c..adbe36538c8a 100644 --- a/components/eamxx/src/share/field/field_impl.hpp +++ b/components/eamxx/src/share/field/field_impl.hpp @@ -222,9 +222,112 @@ get_strided_view_type { return DstView(get_ND_view()); } +template +void Field::sync_views_impl () const { + // For all Kokkos::deep_copy() calls we will pass in an instance of the + // device execution space so that we are asynchronous w.r.t. host. + using DeviceExecSpace = typename Field::get_device::execution_space; + + // Rank 0 will always be contiguous. Copy and return early. + if (rank() == 0) { + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); + return; + } + + const bool is_contiguous = get_header().get_alloc_properties().contiguous(); + if (is_contiguous) { + // For contiguous fields, simply use Kokkos::deep_copy(). + switch (rank()) { + case 1: + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); + break; + case 2: + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); + break; + case 3: + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); + break; + case 4: + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); + break; + case 5: + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); + break; + case 6: + Kokkos::deep_copy(DeviceExecSpace(), get_view(), get_view()); + break; + default: + EKAT_ERROR_MSG ("Error! Unsupported field rank in Field::sync_to_host.\n"); + } + } else { + auto sync_helper = [this] () { + if constexpr (To==Host) m_contiguous_field->sync_to_host(); + else m_contiguous_field->sync_to_dev(); + }; + switch (rank()) { + case 1: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 2: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 3: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 4: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 5: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + case 6: + Kokkos::deep_copy(DeviceExecSpace(), + m_contiguous_field->get_view(), + get_strided_view()); + sync_helper(); + Kokkos::deep_copy(DeviceExecSpace(), + get_strided_view(), + m_contiguous_field->get_view()); + break; + default: + EKAT_ERROR_MSG ("Error! Unsupported field rank in Field::sync_to_host.\n"); + } + } +} + template void Field:: -deep_copy (const Field& src) { +deep_copy (const Field& src) const { EKAT_REQUIRE_MSG (not m_is_read_only, "Error! Cannot call deep_copy on read-only fields.\n"); @@ -248,7 +351,7 @@ deep_copy (const Field& src) { template void Field:: -deep_copy (const ST value) { +deep_copy (const ST value) const { EKAT_REQUIRE_MSG (not m_is_read_only, "Error! Cannot call deep_copy on read-only fields.\n"); @@ -282,7 +385,7 @@ deep_copy (const ST value) { template void Field:: -deep_copy_impl (const Field& src) { +deep_copy_impl (const Field& src) const { const auto& layout = get_header().get_identifier().get_layout(); const auto& layout_src = src.get_header().get_identifier().get_layout(); @@ -306,17 +409,19 @@ deep_copy_impl (const Field& src) { auto src_alloc_props = src.get_header().get_alloc_properties(); auto tgt_alloc_props = get_header().get_alloc_properties(); - // If a manual parallel_for is required (b/c of alloc sizes difference), - // we need to create extents (rather than just using the one in layout), - // since we don't know if we're running on host or device using device_t = typename Field::get_device; using exec_space = typename device_t::execution_space; using RangePolicy = Kokkos::RangePolicy; - using extents_type = typename ekat::KokkosTypes::template view_1d; - extents_type ext ("",rank); - Kokkos::deep_copy(ext,layout.extents()); + auto policy = RangePolicy(0,layout.size()); + using extents_type = typename ekat::KokkosTypes::template view_1d; + extents_type ext; + if constexpr (HD==Device) { + ext = layout.extents(); + } else { + ext = layout.extents_h(); + } switch (rank) { case 1: { @@ -324,14 +429,14 @@ deep_copy_impl (const Field& src) { auto v = get_view< ST*,HD>(); auto v_src = src.get_view(); Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { - v(idx) = v_src(idx); - }); + v(idx) = v_src(idx); + }); } else { auto v = get_strided_view< ST*,HD>(); auto v_src = src.get_strided_view(); Kokkos::parallel_for(policy,KOKKOS_LAMBDA(const int idx) { - v(idx) = v_src(idx); - }); + v(idx) = v_src(idx); + }); } } break; @@ -426,7 +531,7 @@ deep_copy_impl (const Field& src) { } template -void Field::deep_copy_impl (const ST value) { +void Field::deep_copy_impl (const ST value) const { // Note: we can't just do a deep copy on get_view_impl(), since this // field might be a subfield of another. Instead, get the @@ -661,14 +766,16 @@ update_impl (const Field& x, const ST alpha, const ST beta, const ST fill_val) using device_t = typename Field::get_device; using exec_space = typename device_t::execution_space; using RangePolicy = Kokkos::RangePolicy; + auto policy = RangePolicy(0,x_l.size()); - // Need to create extents (rather than just using the one in x_l), - // since we don't know if we're running on host or device using extents_type = typename ekat::KokkosTypes::template view_1d; - extents_type ext ("",x_l.rank()); - Kokkos::deep_copy(ext,x_l.extents()); + extents_type ext; + if constexpr (HD==Device) { + ext = x_l.extents(); + } else { + ext = x_l.extents_h(); + } - auto policy = RangePolicy(0,x_l.size()); switch (x_l.rank()) { case 0: { diff --git a/components/eamxx/src/share/field/field_layout.cpp b/components/eamxx/src/share/field/field_layout.cpp index 2ce924cd0a74..7cd9e953f657 100644 --- a/components/eamxx/src/share/field/field_layout.cpp +++ b/components/eamxx/src/share/field/field_layout.cpp @@ -250,9 +250,9 @@ FieldLayout& FieldLayout::reset_dim (const FieldTag t, const int extent, const b void FieldLayout::set_extents () { m_extents = decltype(m_extents)("",m_rank); - auto extents_h = Kokkos::create_mirror_view(m_extents); - std::copy_n(m_dims.begin(),m_rank,extents_h.data()); - Kokkos::deep_copy(m_extents,extents_h); + m_extents_h = Kokkos::create_mirror_view(m_extents); + std::copy_n(m_dims.begin(),m_rank,m_extents_h.data()); + Kokkos::deep_copy(m_extents,m_extents_h); } void FieldLayout::compute_type () { diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index 018831f8818c..481ca0c922e8 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -97,6 +97,7 @@ class FieldLayout { int dim (const int idim) const; const std::vector& dims () const { return m_dims; } const extents_type& extents () const { return m_extents; } + const extents_type::HostMirror& extents_h () const { return m_extents_h; } long long size () const; @@ -156,6 +157,7 @@ class FieldLayout { std::vector m_names; std::vector m_dims; extents_type m_extents; + extents_type::HostMirror m_extents_h; LayoutType m_type; }; diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index e7e11ef20e2a..c930013f8398 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -4,6 +4,7 @@ #include "share/grid/remap/do_nothing_remapper.hpp" #include "share/property_checks/field_nan_check.hpp" #include "share/property_checks/field_within_interval_check.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include "share/io/scorpio_input.hpp" #include "physics/share/physics_constants.hpp" diff --git a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp index f915cb14d292..7d6590c327a6 100644 --- a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp @@ -3,6 +3,7 @@ #include "share/grid/point_grid.hpp" #include "share/grid/grid_import_export.hpp" #include "share/io/scorpio_input.hpp" +#include "share/field/field.hpp" #include #include diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index 5e714c48aa0d..9b524cec5e49 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -5,8 +5,9 @@ #include "share/field/field_tag.hpp" #include "share/field/field_identifier.hpp" #include "share/util/scream_universal_constants.hpp" +#include "share/io/scream_scorpio_interface.hpp" -#include "ekat/util/ekat_units.hpp" +#include #include #include #include diff --git a/components/eamxx/src/share/io/CMakeLists.txt b/components/eamxx/src/share/io/CMakeLists.txt index afb0ef888db7..6c24d3bc51da 100644 --- a/components/eamxx/src/share/io/CMakeLists.txt +++ b/components/eamxx/src/share/io/CMakeLists.txt @@ -54,6 +54,7 @@ endif() add_library(scream_io scream_output_manager.cpp scorpio_input.cpp + scorpio_scm_input.cpp scorpio_output.cpp scream_io_utils.cpp ) diff --git a/components/eamxx/src/share/io/scorpio_input.hpp b/components/eamxx/src/share/io/scorpio_input.hpp index 801d224be831..2c0a7e76c3eb 100644 --- a/components/eamxx/src/share/io/scorpio_input.hpp +++ b/components/eamxx/src/share/io/scorpio_input.hpp @@ -1,10 +1,8 @@ #ifndef SCREAM_SCORPIO_INPUT_HPP #define SCREAM_SCORPIO_INPUT_HPP -#include "share/io/scream_scorpio_interface.hpp" #include "share/field/field_manager.hpp" #include "share/grid/abstract_grid.hpp" -#include "share/grid/grids_manager.hpp" #include "ekat/ekat_parameter_list.hpp" #include "ekat/logging/ekat_logger.hpp" @@ -23,22 +21,11 @@ * ----- * Input Parameters * Filename: STRING - * Fields: ARRAY OF STRINGS + * Field Names: ARRAY OF STRINGS * ----- * The meaning of these parameters is the following: * - Filename: the name of the input file to be read. - * - Fields: list of names of fields to load from file. Should match the name in the file and the name in the field manager. - * Note: you can specify lists (such as the 'Fields' list above) with either of the two syntaxes - * Fields: [field_name1, field_name2, ... , field_name_N] - * Fields: - * - field_name_1 - * - field_name_2 - * ... - * - field_name_N - * Note: an alternative way of specifying Fields names is to have - * Grid: STRING - * Fields: - * $GRID: [field_name1,...,field_name_N] + * - Field Names: list of names of fields to load from file. Should match the name in the file and the name in the field manager. * * TODO: add a rename option if variable names differ in file and field manager. * @@ -55,8 +42,6 @@ class AtmosphereInput public: using fm_type = FieldManager; using grid_type = AbstractGrid; - using gm_type = GridsManager; - using remapper_type = AbstractRemapper; using KT = KokkosTypes; template diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index e79d75e97b13..a105e244d76f 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1038,11 +1038,42 @@ register_variables(const std::string& filename, } // Now register the average count variables if (m_track_avg_cnt) { + std::string unitless = "unitless"; for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); auto vec_of_dims = set_vec_of_dims(layout); - scorpio::define_var(filename, name, "unitless", vec_of_dims, - "real",fp_precision, m_add_time_dim); + if (mode==scorpio::FileMode::Append) { + // Similar to the regular fields above, check that the var is in the file, and has the right properties + EKAT_REQUIRE_MSG (scorpio::has_var(filename,name), + "Error! Cannot append, due to variable missing from the file.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n"); + const auto& var = scorpio::get_var(filename,name); + EKAT_REQUIRE_MSG (var.dim_names()==vec_of_dims, + "Error! Cannot append, due to variable dimensions mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var dims : " + ekat::join(vec_of_dims,",") + "\n" + " - var dims from file: " + ekat::join(var.dim_names(),",") + "\n"); + EKAT_REQUIRE_MSG (var.units==unitless, + "Error! Cannot append, due to variable units mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var units: " + unitless + "\n" + " - var units from file: " + var.units + "\n"); + EKAT_REQUIRE_MSG (var.time_dep==m_add_time_dim, + "Error! Cannot append, due to time dependency mismatch.\n" + " - filename : " + filename + "\n" + " - varname : " + name + "\n" + " - var time dep: " + (m_add_time_dim ? "yes" : "no") + "\n" + " - var time dep from file: " + (var.time_dep ? "yes" : "no") + "\n"); + } else { + // Note, unlike with regular output variables, for the average counting + // variables we don't need to add all of the extra metadata. So we simply + // define the variable. + scorpio::define_var(filename, name, unitless, vec_of_dims, + "real",fp_precision, m_add_time_dim); + } } } } // register_variables @@ -1378,6 +1409,12 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { diag_name = "VaporFlux"; // split will return the list [X, ''], with X being whatever is before 'VapFlux' params.set("Wind Component",ekat::split(diag_field_name,"VapFlux").front()); + } else if (diag_field_name.find("_atm_backtend")!=std::string::npos) { + diag_name = "AtmBackTendDiag"; + // Set the grid_name + params.set("grid_name",get_field_manager("sim")->get_grid()->name()); + // split will return [X, ''], with X being whatever is before '_atm_tend' + params.set("Tendency Name",ekat::split(diag_field_name,"_atm_backtend").front()); } else if (diag_field_name=="PotentialTemperature" or diag_field_name=="LiqPotentialTemperature") { diag_name = "PotentialTemperature"; diff --git a/components/eamxx/src/share/io/scorpio_scm_input.cpp b/components/eamxx/src/share/io/scorpio_scm_input.cpp new file mode 100644 index 000000000000..c5a4b9149ae9 --- /dev/null +++ b/components/eamxx/src/share/io/scorpio_scm_input.cpp @@ -0,0 +1,246 @@ +#include "share/io/scorpio_scm_input.hpp" +#include "share/io/scorpio_input.hpp" + +#include "share/io/scream_scorpio_interface.hpp" +#include "share/grid/point_grid.hpp" + +#include + +#include + +namespace scream +{ + +SCMInput:: +SCMInput (const std::string& filename, + const double lat, const double lon, + const std::vector& fields, + const ekat::Comm& comm) + : m_comm(comm) + , m_filename (filename) +{ + auto iotype = scorpio::str2iotype("default"); + scorpio::register_file(m_filename,scorpio::Read,iotype); + + // Some input files have the "time" dimension as non-unlimited. This messes up our + // scorpio interface. To avoid trouble, if a dim called 'time' is present we + // treat it as unlimited, even though it isn't. + if (scorpio::has_dim(m_filename,"time") and not scorpio::is_dim_unlimited(m_filename,"time")) { + scorpio::pretend_dim_is_unlimited(m_filename,"time"); + } + + create_io_grid (); + auto ncols = m_io_grid->get_num_local_dofs(); + + create_closest_col_info (lat, lon); + + // Init fields specs + for (const auto& f : fields) { + const auto& fh = f.get_header(); + const auto& fid = fh.get_identifier(); + const auto& fl = fid.get_layout(); + + EKAT_REQUIRE_MSG (fl.tags()[0]==FieldTag::Column, + "Error! SCMInput only works for physics-type layouts.\n" + " - field name: " + f.name() + "\n" + " - field layout: " + fl.to_string() + "\n"); + + m_fields.push_back(f.subfield(0,0)); + FieldIdentifier fid_io(f.name(),fl.clone().reset_dim(0,ncols),fid.get_units(),m_io_grid->name()); + auto& f_io = m_io_fields.emplace_back(fid_io); + f_io.allocate_view(); + } + + // Init scorpio internal structures + init_scorpio_structures (); +} + +SCMInput:: +~SCMInput () +{ + scorpio::release_file(m_filename); +} + +void SCMInput::create_io_grid () +{ + EKAT_REQUIRE_MSG (scorpio::has_dim(m_filename,"ncol"), + "Error! Dimension 'ncol' not found in input file.\n" + " - filename: " + m_filename + "\n"); + const int ncols = scorpio::get_dimlen(m_filename,"ncol"); + const int nlevs = scorpio::has_dim(m_filename,"lev") ? scorpio::get_dimlen(m_filename,"lev") : 1; + + m_io_grid = create_point_grid("scm_io_grid",ncols,nlevs,m_comm); +} + +void SCMInput::create_closest_col_info (double target_lat, double target_lon) +{ + // Read lat/lon fields + const auto ncols = m_io_grid->get_num_local_dofs(); + + auto nondim = ekat::units::Units::nondimensional(); + auto lat = m_io_grid->create_geometry_data("lat",m_io_grid->get_2d_scalar_layout(),nondim); + auto lon = m_io_grid->create_geometry_data("lon",m_io_grid->get_2d_scalar_layout(),nondim); + + // Read from file + AtmosphereInput file_reader(m_filename, m_io_grid, {lat,lon}); + file_reader.read_variables(); + file_reader.finalize(); + + // Find column index of closest lat/lon to target_lat/lon params + auto lat_d = lat.get_view(); + auto lon_d = lon.get_view(); + using minloc_t = Kokkos::MinLoc; + using minloc_value_t = typename minloc_t::value_type; + minloc_value_t minloc; + Kokkos::parallel_reduce(ncols, KOKKOS_LAMBDA (int icol, minloc_value_t& result) { + auto dist = std::abs(lat_d(icol)-target_lat)+std::abs(lon_d(icol)-target_lon); + if(dist min_dist_and_rank = {minloc.val, my_rank}; + m_comm.all_reduce>(&min_dist_and_rank, 1, MPI_MINLOC); + + // Set local col idx to -1 for mpi ranks not containing minimum lat/lon distance + m_closest_col_info.mpi_rank = min_dist_and_rank.second; + m_closest_col_info.col_lid = my_rank==min_dist_and_rank.second ? minloc.loc : -1; +} + +void SCMInput::read_variables (const int time_index) +{ + auto func_start = std::chrono::steady_clock::now(); + auto fname = [](const Field& f) { return f.name(); }; + if (m_atm_logger) { + m_atm_logger->info("[EAMxx::scorpio_scm_input] Reading variables from file"); + m_atm_logger->info(" file name: " + m_filename); + m_atm_logger->info(" var names: " + ekat::join(m_fields,fname,", ")); + if (time_index!=-1) { + m_atm_logger->info(" time idx : " + std::to_string(time_index)); + } + } + + // MPI rank with closest column index store column data + const int n = m_fields.size(); + for (int i=0; i(),time_index); + + auto& f = m_fields[i]; + if (m_comm.rank() == m_closest_col_info.mpi_rank) { + // This rank read the column we need, so copy it in the output field + f.deep_copy(f_io.subfield(0,m_closest_col_info.col_lid)); + } + + // Broadcast column data to all other ranks + const auto col_size = f.get_header().get_identifier().get_layout().size(); + m_comm.broadcast(f.get_internal_view_data(), col_size, m_closest_col_info.mpi_rank); + + // Sync fields to device + f.sync_to_dev(); + } + + auto func_finish = std::chrono::steady_clock::now(); + if (m_atm_logger) { + auto duration = std::chrono::duration_cast(func_finish - func_start)/1000.0; + m_atm_logger->info(" Done! Elapsed time: " + std::to_string(duration.count()) +" seconds"); + } +} + +void SCMInput::init_scorpio_structures() +{ + using namespace ShortFieldTagsNames; + + // Check variables are in the input file + for (const auto& f : m_io_fields) { + const auto& layout = f.get_header().get_identifier().get_layout(); + auto dim_names = layout.names(); + for (int i=0; iget_partitioned_dim_global_size() : layout.dim(i); + EKAT_REQUIRE_MSG (eamxx_len==file_len, + "Error! Dimension mismatch for input file variable.\n" + " - filename : " + m_filename + "\n" + " - varname : " + f.name() + "\n" + " - var dims : " + ekat::join(dim_names,",") + "\n" + " - dim name : " + dim_names[i] + "\n" + " - expected extent : " + std::to_string(eamxx_len) + "\n" + " - extent from file: " + std::to_string(file_len) + "\n"); + } + + // Ensure that we can read the var using Real data type + scorpio::change_var_dtype (m_filename,f.name(),"real"); + } + + // Set decompositions for the variables + set_decompositions(); +} + +/* ---------------------------------------------------------- */ +void SCMInput::set_decompositions() +{ + using namespace ShortFieldTagsNames; + + // First, check if any of the vars is indeed partitioned + const auto decomp_tag = m_io_grid->get_partitioned_dim_tag(); + + bool has_decomposed_layouts = false; + for (const auto& f : m_io_fields) { + const auto& layout = f.get_header().get_identifier().get_layout(); + if (layout.has_tag(decomp_tag)) { + has_decomposed_layouts = true; + break; + } + } + if (not has_decomposed_layouts) { + // If none of the input vars are decomposed on this grid, + // then there's nothing to do here + return; + } + + // Set the decomposition for the partitioned dimension + const int local_dim = m_io_grid->get_partitioned_dim_local_size(); + std::string decomp_dim = m_io_grid->has_special_tag_name(decomp_tag) + ? m_io_grid->get_special_tag_name(decomp_tag) + : e2str(decomp_tag); + + auto gids_f = m_io_grid->get_partitioned_dim_gids(); + auto gids_h = gids_f.get_view(); + auto min_gid = m_io_grid->get_global_min_partitioned_dim_gid(); + std::vector offsets(local_dim); + for (int idof=0; idof + +namespace scream +{ + +// Similar to AtmosphereInput, but reads in a single column from +// a file with N columns. A few assumptions: +// - lat and lon variables are present in the file +// - fields have layout +class SCMInput +{ +public: + // --- Constructor(s) & Destructor --- // + SCMInput (const std::string& filename, + const double lat, const double lon, + const std::vector& fields, + const ekat::Comm& comm); + + ~SCMInput (); + + // Due to resource acquisition (in scorpio), avoid copies + SCMInput (const SCMInput&) = delete; + SCMInput& operator= (const SCMInput&) = delete; + + // Read fields that were required via parameter list. + void read_variables (const int time_index = -1); + + // Option to add a logger + void set_logger(const std::shared_ptr& atm_logger) { + m_atm_logger = atm_logger; + } + +#ifndef KOKKOS_ENABLE_CUDA + // Cuda requires methods enclosing __device__ lambda's to be public +protected: +#endif + void create_closest_col_info (double target_lat, double target_lon); +protected: + + struct ClosestColInfo { + // MPI rank which owns the columns whose lat/lon pair is the closest to target lat/lon + int mpi_rank; + // Local column index of on rank=mpi_rank (-1 on all other ranks) + int col_lid; + }; + + void create_io_grid (); + void init_scorpio_structures (); + void set_decompositions(); + + // Internal variables + ekat::Comm m_comm; + + std::shared_ptr m_io_grid; + + std::string m_filename; + std::vector m_fields; + std::vector m_io_fields; + + ClosestColInfo m_closest_col_info; + + // The logger to be used throughout the ATM to log message + std::shared_ptr m_atm_logger; +}; + +} //namespace scream + +#endif // SCREAM_SCORPIO_SCM_INPUT_HPP diff --git a/components/eamxx/src/share/io/scream_io_utils.cpp b/components/eamxx/src/share/io/scream_io_utils.cpp index 4a0beb813b76..9318728d657a 100644 --- a/components/eamxx/src/share/io/scream_io_utils.cpp +++ b/components/eamxx/src/share/io/scream_io_utils.cpp @@ -2,8 +2,10 @@ #include "share/io/scream_scorpio_interface.hpp" #include "share/util/scream_utils.hpp" +#include "share/scream_config.hpp" #include +#include namespace scream { @@ -11,37 +13,49 @@ std::string find_filename_in_rpointer ( const std::string& filename_prefix, const bool model_restart, const ekat::Comm& comm, - const util::TimeStamp& run_t0) + const util::TimeStamp& run_t0, + const OutputAvgType avg_type, + const IOControl& control) { std::string filename; bool found = false; std::string content; std::string suffix = model_restart ? ".r." : ".rhist."; + std::string pattern_str = filename_prefix + suffix; + + // The AD will pass a default constructed control, since it doesn't know the values + // of REST_N/REST_OPTION used in the previous run. Also, model restart is *always* INSTANT. + if (model_restart) { + EKAT_REQUIRE_MSG (avg_type==OutputAvgType::Instant, + "Error! Model restart output should have INSTANT avg type.\n" + " - input avg_type: " + e2str(avg_type) + "\n"); + pattern_str += e2str(OutputAvgType::Instant) + R"(.n(step|sec|min|hour|day|month|year)s_x\d+)"; + } else { + EKAT_REQUIRE_MSG (control.output_enabled(), + "Error! When restarting an output stream, we need a valid IOControl structure.\n" + " - filename prefix: " + filename_prefix + "\n"); + pattern_str += e2str(avg_type) + "." + control.frequency_units + "_x" + std::to_string(control.frequency); + } + if (is_scream_standalone()) { + pattern_str += ".np" + std::to_string(comm.size()); + } + pattern_str += "." + run_t0.to_string() + ".nc"; + std::regex pattern (pattern_str); + if (comm.am_i_root()) { std::ifstream rpointer_file; + std::string line; rpointer_file.open("rpointer.atm"); - // If the timestamp is in the filename, then the filename ends with "S.nc", - // with S being the string representation of the timestamp - auto ts_len = run_t0.to_string().size(); - auto extract_ts = [&] (const std::string& line) -> util::TimeStamp { - auto min_size = ts_len+3; - if (line.size()>=min_size) { - auto ts_str = line.substr(line.size()-min_size,ts_len); - auto ts = util::str_to_time_stamp(ts_str); - return ts; - } else { - return util::TimeStamp(); - } - }; - - while ((rpointer_file >> line) and not found) { + while (std::getline(rpointer_file,line)) { content += line + "\n"; - found = line.find(filename_prefix+suffix) != std::string::npos && - extract_ts(line)==run_t0; - filename = line; + if (std::regex_search(line,pattern)) { + filename = line; + found = true; + break; + } } } @@ -52,18 +66,29 @@ std::string find_filename_in_rpointer ( if (not found) { broadcast_string(content,comm,comm.root_rank()); - // If the history restart file is not found, it must be because the last - // model restart step coincided with a model output step, in which case - // a restart history file is not written. - // If that's the case, *disable* output restart, by setting - // 'Restart'->'Perform Restart' = false - // in the input parameter list - EKAT_ERROR_MSG ( - "Error! Restart requested, but no restart file found in 'rpointer.atm'.\n" - " restart filename prefix: " + filename_prefix + "\n" - " restart file type: " + std::string(model_restart ? "model restart" : "history restart") + "\n" - " run t0 : " + run_t0.to_string() + "\n" - " rpointer content:\n" + content); + if (model_restart) { + EKAT_ERROR_MSG ( + "Error! Restart requested, but no model restart file found in 'rpointer.atm'.\n" + " model restart filename prefix: " + filename_prefix + "\n" + " model restart filename pattern: " + pattern_str + "\n" + " run t0 : " + run_t0.to_string() + "\n" + " rpointer content:\n" + content + "\n\n"); + } else { + EKAT_ERROR_MSG ( + "Error! Restart requested, but no history restart file found in 'rpointer.atm'.\n" + " hist restart filename prefix: " + filename_prefix + "\n" + " hist restart filename pattern: " + pattern_str + "\n" + " run t0 : " + run_t0.to_string() + "\n" + " avg_type : " + e2str(avg_type) + "\n" + " output freq : " + std::to_string(control.frequency) + "\n" + " output freq units: " + control.frequency_units + "\n" + " rpointer content:\n" + content + "\n\n" + " Did you change output specs (avg type, freq, or freq units) across restart? If so, please, remember that it is not allowed.\n" + " It is also possible you are using a rhist file create before commit 6b7d441330d. That commit changed how rhist file names\n" + " are formed. In particular, we no longer use INSTANT.${REST_OPTION}_x${REST_N}, but we use the avg type, and freq/freq_option\n" + " of the output stream (to avoid name clashes if 2 streams only differ for one of those). If you want to use your rhist file,\n" + " please rename it, so that the avg-type, freq, and freq_option reflect those of the output stream.\n"); + } } // Have the root rank communicate the nc filename diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index efb2a4fd65bb..01bc46e1e601 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -1,6 +1,7 @@ #ifndef SCREAM_IO_UTILS_HPP #define SCREAM_IO_UTILS_HPP +#include "scream_io_control.hpp" #include "share/util/scream_time_stamp.hpp" #include @@ -59,11 +60,17 @@ inline OutputAvgType str2avg (const std::string& s) { return OAT::Invalid; } +// The AD will pass a default constructed control, since it doesn't know the values +// of REST_N/REST_OPTION used in the previous run +// Output streams MUST pass a valid control structure, cause we need to differentiate +// between, e.g., streams with same filename prefix, but different output freq specs std::string find_filename_in_rpointer ( - const std::string& casename, + const std::string& filename_prefix, const bool model_restart, const ekat::Comm& comm, - const util::TimeStamp& run_t0); + const util::TimeStamp& run_t0, + const OutputAvgType avg_type = OutputAvgType::Instant, + const IOControl& control = {}); struct LongNames { diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 35b1c753b03e..4f35827ba3f4 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -171,7 +171,9 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, if (perform_history_restart) { using namespace scorpio; - auto rhist_file = find_filename_in_rpointer(hist_restart_filename_prefix,false,m_io_comm,m_run_t0); + IOFileSpecs hist_restart_specs; + hist_restart_specs.ftype = FileType::HistoryRestart; + auto rhist_file = find_filename_in_rpointer(hist_restart_filename_prefix,false,m_io_comm,m_run_t0,m_avg_type,m_output_control); scorpio::register_file(rhist_file,scorpio::Read); // From restart file, get the time of last write, as well as the current size of the avg sample @@ -196,22 +198,8 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // We do NOT allow changing output specs across restart. If you do want to change // any of these, you MUST start a new output stream (e.g., setting 'Perform Restart: false') - auto old_freq = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_frequency"); - EKAT_REQUIRE_MSG (old_freq == m_output_control.frequency, - "Error! Cannot change frequency when performing history restart.\n" - " - old freq: " << old_freq << "\n" - " - new freq: " << m_output_control.frequency << "\n"); - auto old_freq_units = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_frequency_units"); - EKAT_REQUIRE_MSG (old_freq_units == m_output_control.frequency_units, - "Error! Cannot change frequency units when performing history restart.\n" - " - old freq units: " << old_freq_units << "\n" - " - new freq units: " << m_output_control.frequency_units << "\n"); - auto old_avg_type = scorpio::get_attribute(rhist_file,"GLOBAL","averaging_type"); - EKAT_REQUIRE_MSG (old_avg_type == e2str(m_avg_type), - "Error! Cannot change avg type when performing history restart.\n" - " - old avg type: " << old_avg_type + "\n" - " - new avg type: " << e2str(m_avg_type) << "\n"); - + // NOTE: we do not check that freq/freq_units/avg_type are not changed: since we used + // that info to find the correct rhist file, we already know that they match! auto old_storage_type = scorpio::get_attribute(rhist_file,"GLOBAL","file_max_storage_type"); EKAT_REQUIRE_MSG (old_storage_type == e2str(m_output_file_specs.storage.type), "Error! Cannot change file storage type when performing history restart.\n" @@ -242,17 +230,18 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, const auto& last_output_filename = get_attribute(rhist_file,"GLOBAL","last_output_filename"); m_resume_output_file = last_output_filename!="" and not restart_pl.get("force_new_file",false); if (m_resume_output_file) { - scorpio::register_file(last_output_filename,scorpio::Read,m_output_file_specs.iotype); - int num_snaps = scorpio::get_dimlen(last_output_filename,"time"); - scorpio::release_file(last_output_filename); + int num_snaps = scorpio::get_attribute(rhist_file,"GLOBAL","last_output_file_num_snaps"); m_output_file_specs.filename = last_output_filename; m_output_file_specs.is_open = true; m_output_file_specs.storage.num_snapshots_in_file = num_snaps; - // The setup_file call will not register any new variable (the file is in Append mode, - // so all dims/vars must already be in the file). However, it will register decompositions, - // since those are a property of the run, not of the file. - setup_file(m_output_file_specs,m_output_control); + + if (m_output_file_specs.storage.snapshot_fits(m_output_control.next_write_ts)) { + // The setup_file call will not register any new variable (the file is in Append mode, + // so all dims/vars must already be in the file). However, it will register decompositions, + // since those are a property of the run, not of the file. + setup_file(m_output_file_specs,m_output_control); + } } scorpio::release_file(rhist_file); } @@ -328,6 +317,17 @@ void OutputManager::run(const util::TimeStamp& timestamp) return; } + // Ensure we did not go past the scheduled write time without hitting it + EKAT_REQUIRE_MSG ( + (m_output_control.frequency_units=="nsteps" + ? timestamp.get_num_steps()<=m_output_control.next_write_ts.get_num_steps() + : timestamp<=m_output_control.next_write_ts), + "Error! The input timestamp is past the next scheduled write timestamp.\n" + " - current time stamp : " + timestamp.to_string() + "\n" + " - next write time stamp: " + m_output_control.next_write_ts.to_string() + "\n" + "The most likely cause is an output frequency that is faster than the atm timestep.\n" + "Try to increase 'Frequency' and/or 'frequency_units' in your output yaml file.\n"); + // Update counters ++m_output_control.nsamples_since_last_write; ++m_checkpoint_control.nsamples_since_last_write; @@ -381,14 +381,14 @@ void OutputManager::run(const util::TimeStamp& timestamp) snapshot_start = m_case_t0; snapshot_start += m_time_bnds[0]; } - if (not filespecs.storage.snapshot_fits(snapshot_start)) { + if (filespecs.is_open and not filespecs.storage.snapshot_fits(snapshot_start)) { release_file(filespecs.filename); filespecs.close(); } // Check if we need to open a new file if (not filespecs.is_open) { - filespecs.filename = compute_filename (control,filespecs,timestamp); + filespecs.filename = compute_filename (filespecs,timestamp); // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm) setup_file(filespecs,control); } @@ -483,6 +483,10 @@ void OutputManager::run(const util::TimeStamp& timestamp) write_timestamp (filespecs.filename,"last_write",m_output_control.last_write_ts,true); scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_filename",m_output_file_specs.filename); scorpio::set_attribute (filespecs.filename,"GLOBAL","num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); + + int nsnaps = m_output_file_specs.is_open + ? scorpio::get_dimlen(m_output_file_specs.filename,"time") : 0; + scorpio::set_attribute (filespecs.filename,"GLOBAL","last_output_file_num_snaps",nsnaps); } // Write these in both output and rhist file. The former, b/c we need these info when we postprocess // output, and the latter b/c we want to make sure these params don't change across restarts @@ -597,18 +601,20 @@ long long OutputManager::res_dep_memory_footprint () const { } std::string OutputManager:: -compute_filename (const IOControl& control, - const IOFileSpecs& file_specs, +compute_filename (const IOFileSpecs& file_specs, const util::TimeStamp& timestamp) const { auto filename = m_filename_prefix + file_specs.suffix(); + const auto& control = m_output_control; // Always add avg type and frequency info filename += "." + e2str(m_avg_type); filename += "." + control.frequency_units+ "_x" + std::to_string(control.frequency); - // Optionally, add number of mpi ranks (useful mostly in unit tests, to run multiple MPI configs in parallel) - if (m_params.get("MPI Ranks in Filename")) { + // For standalone EAMxx, we may have 2+ versions of the same test running with two + // different choices of ranks. To avoid name clashing for the output files, + // add the comm size to the output file name. + if (is_scream_standalone()) { filename += ".np" + std::to_string(m_io_comm.size()); } @@ -663,7 +669,6 @@ set_params (const ekat::ParameterList& params, m_filename_prefix = m_params.get("filename_prefix"); // Hard code some parameters in case we access them later - m_params.set("MPI Ranks in Filename",false); m_params.set("Floating Point Precision","real"); } else { auto avg_type = m_params.get("Averaging Type"); @@ -699,12 +704,6 @@ set_params (const ekat::ParameterList& params, "Error! Invalid/unsupported value for 'Floating Point Precision'.\n" " - input value: " + prec + "\n" " - supported values: float, single, double, real\n"); - - // If not set, hard code to false for CIME cases, and true for standalone, - // since standalone may be running multiple versions of the same test at once - if (not m_params.isParameter("MPI Ranks in Filename")) { - m_params.set("MPI Ranks in Filename",is_scream_standalone()); - } } // Output control @@ -778,6 +777,24 @@ setup_file ( IOFileSpecs& filespecs, auto mode = m_resume_output_file ? scorpio::Append : scorpio::Write; scorpio::register_file(filename,mode,filespecs.iotype); if (m_resume_output_file) { + // We may have resumed an output file that contains extra snapshots *after* the restart time. + // E.g., if we output every step and the run crashed a few steps after writing the restart. + // In that case, we need to reset the time dimension in the output file, so that the extra + // snapshots will be overwritten. + const auto all_times = scorpio::get_all_times(filename); + int ntimes = all_times.size(); + int ngood = 0; + for (const auto& t : all_times) { + auto keep = t<=m_output_control.last_write_ts.days_from(m_case_t0); + if (keep) { + ++ngood; + } else { + break; + } + } + if (ngood(PIO_IOTYPE_NETCDF); break; + case IOType::PnetCDF: iotype_int = static_cast(PIO_IOTYPE_PNETCDF); break; + case IOType::Adios: iotype_int = static_cast(PIO_IOTYPE_ADIOS); break; + case IOType::Hdf5: iotype_int = static_cast(PIO_IOTYPE_HDF5); break; + default: + EKAT_ERROR_MSG ("Unrecognized/unsupported iotype.\n"); + } + return iotype_int; +} + // ====================== Local utilities ========================== // namespace impl { @@ -313,9 +328,9 @@ void init_subsystem(const ekat::Comm& comm, const int atm_id) s.pio_rearranger = PIO_REARR_SUBSET; s.pio_format = PIO_64BIT_DATA; #if PIO_USE_PNETCDF - s.pio_type_default = PIO_IOTYPE_PNETCDF; + s.pio_type_default = pio_iotype(IOType::PnetCDF); #elif PIO_USE_NETCDF - s.pio_type_default = PIO_IOTYPE_NETCDF; + s.pio_type_default = pio_iotype(IOType::NetCDF); #else #error "Standalone EAMxx requires either PNETCDF or NETCDF iotype to be available in Scorpio" #endif @@ -395,9 +410,7 @@ void register_file (const std::string& filename, if (f.mode == Unset) { // First time we ask for this file. Call PIO open routine(s) int err; - int iotype_int = iotype==IOType::DefaultIOType - ? s.pio_type_default - : static_cast(iotype); + int iotype_int = pio_iotype(iotype); if (mode & Read) { auto write = mode & Write ? PIO_WRITE : PIO_NOWRITE; err = PIOc_openfile(s.pio_sysid,&f.ncid,&iotype_int,filename.c_str(),write); @@ -704,6 +717,30 @@ std::string get_time_name (const std::string& filename) return pf.file->time_dim->name; } +void reset_unlimited_dim_len(const std::string& filename, const int new_length) +{ + auto& f = impl::get_file(filename,"scorpio::reset_unlimited_dim_len"); + + // Reset dim length + EKAT_REQUIRE_MSG (f.time_dim!=nullptr, + "Error! Cannot reset unlimited dim length. No unlimited dim stored.\n" + " - file name: " + filename + "\n"); + EKAT_REQUIRE_MSG (new_lengthlength, + "Error! New time dimension length must be shorter than the current one.\n" + " - file name: " + filename + "\n" + " - curr len : " + std::to_string(f.time_dim->length) + "\n" + " - new len : " + std::to_string(new_length) + "\n"); + f.time_dim->length = new_length; + + // Reset number of records counter for each time dep var + for (auto it : f.vars) { + auto& v = *it.second; + if (v.time_dep) { + v.num_records = new_length; + } + } +} + // =================== Decompositions operations ==================== // // NOTES: diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index 549de680fe45..54e3d9027651 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -108,6 +108,7 @@ bool is_dim_unlimited (const std::string& filename, // NOTE: these throw if time dim is not present. Use has_dim to check first. int get_time_len (const std::string& filename); std::string get_time_name (const std::string& filename); +void reset_unlimited_dim_len(const std::string& filename, const int new_length); // =================== Decompositions operations ==================== // diff --git a/components/eamxx/src/share/io/scream_scorpio_types.hpp b/components/eamxx/src/share/io/scream_scorpio_types.hpp index 645b1b26937b..3b16cebc06c8 100644 --- a/components/eamxx/src/share/io/scream_scorpio_types.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_types.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace scream { namespace scorpio { diff --git a/components/eamxx/src/share/io/tests/CMakeLists.txt b/components/eamxx/src/share/io/tests/CMakeLists.txt index 16ce2efd4376..8c819f7896cc 100644 --- a/components/eamxx/src/share/io/tests/CMakeLists.txt +++ b/components/eamxx/src/share/io/tests/CMakeLists.txt @@ -81,3 +81,9 @@ CreateUnitTest(io_remap_test "io_remap_test.cpp" LIBS scream_io diagnostics LABELS io remap MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) + +## Test single-column reader +CreateUnitTest(io_scm_reader "io_scm_reader.cpp" + LIBS scream_io LABELS io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} +) diff --git a/components/eamxx/src/share/io/tests/io_scm_reader.cpp b/components/eamxx/src/share/io/tests/io_scm_reader.cpp new file mode 100644 index 000000000000..111f82947504 --- /dev/null +++ b/components/eamxx/src/share/io/tests/io_scm_reader.cpp @@ -0,0 +1,136 @@ +#include + +#include "share/io/scorpio_scm_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +#include "share/grid/point_grid.hpp" +#include "share/field/field.hpp" +#include "share/field/field_utils.hpp" + +#include "share/util/scream_setup_random_test.hpp" + +namespace scream { + +// Returns fields after initialization +void write (const int seed, const int ncols, const int nlevs, const ekat::Comm& comm) +{ + using ekat::units::Units; + using RPDF = std::uniform_real_distribution; + using Engine = std::mt19937_64; + + Engine engine(seed); + RPDF lat_pdf(-90.0,90.0); + RPDF lon_pdf(-180.0,180.0); + + // Create grid + auto grid = create_point_grid("test",ncols,nlevs,comm); + + // Create lat/lon grid + Units deg (Units::nondimensional(),"deg"); + auto lat = grid->create_geometry_data("lat",grid->get_2d_scalar_layout(),deg); + auto lon = grid->create_geometry_data("lon",grid->get_2d_scalar_layout(),deg); + randomize(lat,engine,lat_pdf); + randomize(lon,engine,lon_pdf); + + // Create variable data + FieldIdentifier fid("var",grid->get_3d_scalar_layout(true),Units::nondimensional(),""); + Field var(fid); + var.allocate_view(); + randomize(var,engine,RPDF(-1.0,1.0)); + + // Create file + auto filename = "io_scm_np" + std::to_string(comm.size()) + ".nc"; + scorpio::register_file(filename,scorpio::Write,scorpio::DefaultIOType); + + scorpio::define_dim(filename,"ncol",ncols); + scorpio::define_dim(filename,"lev",nlevs); + + scorpio::define_var(filename,"lat",{"ncol"},"real"); + scorpio::define_var(filename,"lon",{"ncol"},"real"); + + scorpio::define_var(filename,"var",{"ncol","lev"},"real"); + + auto my_col_gids = grid->get_partitioned_dim_gids().get_view(); + std::vector my_offsets(my_col_gids.size()); + for (size_t i=0; iget_global_min_partitioned_dim_gid (); + } + scorpio::set_dim_decomp(filename,"ncol",my_offsets); + scorpio::enddef(filename); + + // Write to file + scorpio::write_var(filename,"lat",lat.get_internal_view_data()); + scorpio::write_var(filename,"lon",lon.get_internal_view_data()); + scorpio::write_var(filename,"var",var.get_internal_view_data()); + + scorpio::release_file(filename); +} + +void read (const int seed, const int nlevs, const ekat::Comm& comm) +{ + using ekat::units::Units; + using IPDF = std::uniform_int_distribution; + using Engine = std::mt19937_64; + + Engine engine(seed); + + auto grid = std::make_shared("scm_grid",1,nlevs,comm); + auto dofs_gids = grid->get_dofs_gids(); + dofs_gids.deep_copy(0); + + // Pick a col index, and find its lat/lon from file + auto filename = "io_scm_np" + std::to_string(comm.size()) + ".nc"; + scorpio::register_file(filename,scorpio::Read,scorpio::DefaultIOType); + int ncols = scorpio::get_dimlen(filename,"ncol"); + + std::vector lat(ncols), lon(ncols), var(ncols*nlevs); + scorpio::read_var(filename,"lat",lat.data()); + scorpio::read_var(filename,"lon",lon.data()); + scorpio::read_var(filename,"var",var.data()); + + scorpio::release_file(filename); + + auto tgt_col = IPDF(0,ncols-1)(engine); + comm.broadcast(&tgt_col,1,comm.root_rank()); + + auto tgt_lat = lat[tgt_col]; + auto tgt_lon = lon[tgt_col]; + + // Create field to read + FieldIdentifier fid("var",grid->get_3d_scalar_layout(true),Units::nondimensional(),""); + Field var_f(fid); + var_f.allocate_view(); + + // Read field + SCMInput reader(filename,tgt_lat,tgt_lon,{var_f},comm); + reader.read_variables(); + + // Check + auto var_h = var_f.get_view(); + for (int ilev=0; ilev; + using Engine = std::mt19937_64; + + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::init_subsystem(comm); + + auto seed = get_random_test_seed(&comm); + Engine engine(seed); + int nlevs = IPDF(5,8)(engine); + int ncols = IPDF(4,5)(engine); + + comm.broadcast(&ncols,1,comm.root_rank()); + comm.broadcast(&nlevs,1,comm.root_rank()); + + write(seed,ncols,nlevs,comm); + read(seed,nlevs,comm); + + scorpio::finalize_subsystem(); +} + +} // anonymous namespace diff --git a/components/eamxx/src/share/io/tests/io_utils.cpp b/components/eamxx/src/share/io/tests/io_utils.cpp index 77779307a720..cf7b66ad8596 100644 --- a/components/eamxx/src/share/io/tests/io_utils.cpp +++ b/components/eamxx/src/share/io/tests/io_utils.cpp @@ -9,6 +9,9 @@ TEST_CASE ("find_filename_in_rpointer") { using namespace scream; + constexpr auto AVG = OutputAvgType::Average; + constexpr auto INST = OutputAvgType::Instant; + ekat::Comm comm(MPI_COMM_WORLD); util::TimeStamp t0({2023,9,7},{12,0,0}); @@ -17,21 +20,35 @@ TEST_CASE ("find_filename_in_rpointer") { // Create a dummy rpointer std::ofstream rpointer ("rpointer.atm"); - rpointer << "foo.r." + t0.to_string() + ".nc\n"; - rpointer << "bar2.rhist." + t0.to_string() + ".nc\n"; - rpointer << "bar.rhist." + t0.to_string() + ".nc\n"; - rpointer.close(); + IOControl foo_c, bar_c, bar2_c; + foo_c.frequency = 3; foo_c.frequency_units = "nsteps"; + bar_c.frequency = 1; bar_c.frequency_units = "ndays"; + bar2_c.frequency = 6; bar2_c.frequency_units = "nhours"; - // Now test find_filename_in_rpointer with different inputs + std::string suffix = ".np" + std::to_string(comm.size()) + "." + t0.to_string() + ".nc"; + std::string foo_fname = "foo.r.INSTANT.nsteps_x3" + suffix; + std::string bar_fname = "bar.rhist.AVERAGE.ndays_x1" + suffix; + std::string bar2_fname = "bar.rhist.AVERAGE.nhours_x6" + suffix; - REQUIRE_THROWS (find_filename_in_rpointer("baz",false,comm,t0)); // wrong prefix - REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t1)); // wrong timestamp - REQUIRE_THROWS (find_filename_in_rpointer("bar",true, comm,t0)); // bar is not model restart - REQUIRE_THROWS (find_filename_in_rpointer("foo",false,comm,t0)); // foo is model restart + rpointer << foo_fname<< "\n"; + rpointer << bar_fname<< "\n"; + rpointer << bar2_fname << "\n"; + rpointer.close(); - REQUIRE (find_filename_in_rpointer("bar", false,comm,t0)==("bar.rhist."+t0.to_string()+".nc")); - REQUIRE (find_filename_in_rpointer("bar2",false,comm,t0)==("bar2.rhist."+t0.to_string()+".nc")); - REQUIRE (find_filename_in_rpointer("foo", true, comm,t0)==("foo.r."+t0.to_string()+".nc")); + // Now test find_filename_in_rpointer with different inputs + REQUIRE_THROWS (find_filename_in_rpointer("baz",false,comm,t0,AVG)); // missing control (needed for rhist files) + REQUIRE_THROWS (find_filename_in_rpointer("baz",false,comm,t0,AVG,bar_c)); // wrong prefix + REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t1,AVG,bar_c)); // wrong timestamp + REQUIRE_THROWS (find_filename_in_rpointer("bar",true, comm,t0,AVG,bar_c)); // bar is not model restart + REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t0,INST,bar_c)); // wrong avg type + REQUIRE_THROWS (find_filename_in_rpointer("bar",false,comm,t0,INST,bar2_c)); // wrong freq specs + REQUIRE_THROWS (find_filename_in_rpointer("foo",false,comm,t0,INST,foo_c)); // foo is model restart + REQUIRE_THROWS (find_filename_in_rpointer("foo",true,comm,t0,AVG)); // model restart MUST be INSTANT + + auto test = find_filename_in_rpointer("bar",false,comm,t0,AVG,bar_c); + REQUIRE (find_filename_in_rpointer("bar",false,comm,t0,AVG,bar_c)==bar_fname); + REQUIRE (find_filename_in_rpointer("bar",false,comm,t0,AVG,bar2_c)==bar2_fname); + REQUIRE (find_filename_in_rpointer("foo",true, comm,t0)==foo_fname); } TEST_CASE ("io_control") { diff --git a/components/eamxx/src/share/iop/intensive_observation_period.cpp b/components/eamxx/src/share/iop/intensive_observation_period.cpp index bbe3306a415e..32b120c57002 100644 --- a/components/eamxx/src/share/iop/intensive_observation_period.cpp +++ b/components/eamxx/src/share/iop/intensive_observation_period.cpp @@ -1,5 +1,6 @@ #include "share/grid/point_grid.hpp" #include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include "share/iop/intensive_observation_period.hpp" #include "ekat/ekat_assert.hpp" @@ -247,15 +248,17 @@ initialize_iop_file(const util::TimeStamp& run_t0, else if (scorpio::has_dim(iop_file, "tsec")) time_dimname = "tsec"; else EKAT_ERROR_MSG("Error! No valid dimension for tsec in "+iop_file+".\n"); - const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); - m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); - scorpio::read_var(iop_file,"tsec",m_time_info.iop_file_times_in_sec.data()); - - // From now on, when we read vars, "time" must be treated as unlimited, to avoid issues + // When we read vars, "time" must be treated as unlimited, to avoid issues if (not scorpio::is_dim_unlimited(iop_file,time_dimname)) { scorpio::pretend_dim_is_unlimited(iop_file,time_dimname); } + const auto ntimes = scorpio::get_dimlen(iop_file, time_dimname); + m_time_info.iop_file_times_in_sec = view_1d_host("iop_file_times", ntimes); + for (int t=0; t("target_latitude"))/ - m_params.get("target_latitude"); + std::max(m_params.get("target_latitude"),(Real)0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ - m_params.get("target_longitude"); + std::max(m_params.get("target_longitude"),(Real)0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), "Error! IOP file variable \"lat\" does not match target_latitude from IOP parameters.\n"); EKAT_REQUIRE_MSG(rel_lon_err < std::numeric_limits::epsilon(), @@ -540,6 +543,14 @@ read_iop_file_data (const util::TimeStamp& current_ts) scorpio::read_var(iop_file,"Ps",ps_data,iop_file_time_idx); surface_pressure.sync_to_dev(); + // Read in IOP lev data + auto data = iop_file_pressure.get_view().data(); + scorpio::read_var(iop_file,"lev",data); + + // Convert to pressure to millibar (file gives pressure in Pa) + for (int ilev=0; ilev(); diff --git a/components/eamxx/src/share/tests/field_tests.cpp b/components/eamxx/src/share/tests/field_tests.cpp index 1cb69f474dee..0dfe2522de39 100644 --- a/components/eamxx/src/share/tests/field_tests.cpp +++ b/components/eamxx/src/share/tests/field_tests.cpp @@ -812,4 +812,78 @@ TEST_CASE ("update") { } } + +TEST_CASE ("sync_subfields") { + // This test is for previously incorrect behavior, where syncing a subfield + // to host/device would deep copy the entire data view (including all entries of + // the parent view). Here, if memory space is not shared between host and device, + // syncing a subfield to host/device will not sync the data of the other subfields. + + using namespace scream; + using namespace ekat::units; + using namespace ShortFieldTagsNames; + using FID = FieldIdentifier; + using FL = FieldLayout; + + constexpr int ncols = 10; + constexpr int ndims = 4; + constexpr int nlevs = 8; + + // Create field with (col, cmp, lev) + FID fid ("V",FL({COL,CMP,LEV},{ncols,ndims,nlevs}),Units::nondimensional(),"the_grid",DataType::IntType); + Field f (fid); + f.allocate_view(); + + // Store whether mem space for host and device are the same for testing subfield values + const bool shared_mem_space = f.host_and_device_share_memory_space(); + + // Deep copy all values to ndims on device and host + f.deep_copy(ndims); + f.sync_to_host(); + + // Set subfield values to their index on device + for (int c=0; c(); + for (int idx=0; idx(c); + } + + // Sync only component 0 to device + f.get_component(0).sync_to_dev(); + + // For components 1,...,ndims-1, if device and host do not share a + // memory space, device values should be equal to ndims, else device + // values should be equal to component index + for (int c=1; c(); + Kokkos::parallel_for(Kokkos::MDRangePolicy>({0,0}, {ncols,nlevs}), + KOKKOS_LAMBDA (const int icol, const int ilev) { + if (shared_mem_space) EKAT_KERNEL_ASSERT(device_subview(icol, ilev) == c); + else EKAT_KERNEL_ASSERT(device_subview(icol, ilev) == ndims); + }); + } +} + } // anonymous namespace diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index e0dba0cf8382..aa46328acac3 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -1,4 +1,5 @@ #include "share/util/eamxx_time_interpolation.hpp" +#include "share/io/scream_scorpio_interface.hpp" #include "share/io/scream_io_utils.hpp" namespace scream{ diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index 5fd23a41d9d8..ad80045a0c13 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -68,10 +68,14 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) # Testing individual atm processes add_subdirectory(single-process) - # Testing multiple atm processes coupled together - add_subdirectory(multi-process) - - if (EAMXX_ENABLE_PYBIND) + # Testing multiple atm processes coupled together. + # Some compute-sanitizer tests time out with these + # larger multiprocess tests, so disable in that case. + if (NOT EKAT_ENABLE_COMPUTE_SANITIZER) + add_subdirectory(multi-process) + endif() + + if (EAMXX_ENABLE_PYSCREAM) add_subdirectory(python) endif() endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt index 64ac2187c280..841b7ea00c64 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt @@ -13,7 +13,8 @@ if (SCREAM_DOUBLE_PRECISION) # initial conditions. #add_subdirectory(homme_mam4xx_pg2) add_subdirectory(mam/homme_shoc_cld_p3_mam_optics_rrtmgp) + add_subdirectory(mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep) + add_subdirectory(mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav) endif() endif() endif() - diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt new file mode 100644 index 000000000000..e3c562cfcd13 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/CMakeLists.txt @@ -0,0 +1,109 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Get or create the dynamics lib +# HOMME_TARGET NP PLEV QSIZE_D +CreateDynamicsLib("theta-l_kokkos" 4 72 41) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS cld_fraction ${dynLibName} shoc p3 scream_rrtmgp mam + LABELS dynamics shoc cld p3 rrtmgp physics mam4_optics mam4_aci mam4_drydep + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Set homme's test options, so that we can configure the namelist correctly +# Discretization/algorithm settings +set (HOMME_TEST_NE 2) +set (HOMME_TEST_LIM 9) +set (HOMME_TEST_REMAP_FACTOR 3) +set (HOMME_TEST_TRACERS_FACTOR 1) +set (HOMME_TEST_TIME_STEP 300) +set (HOMME_THETA_FORM 1) +set (HOMME_TTYPE 5) +set (HOMME_SE_FTYPE 0) +set (HOMME_TEST_TRANSPORT_ALG 0) +set (HOMME_TEST_CUBED_SPHERE_MAP 0) + +# Hyperviscosity settings +set (HOMME_TEST_HVSCALING 0) +set (HOMME_TEST_HVS 1) +set (HOMME_TEST_HVS_TOM 0) +set (HOMME_TEST_HVS_Q 1) + +set (HOMME_TEST_NU 7e15) +set (HOMME_TEST_NUDIV 1e15) +set (HOMME_TEST_NUTOP 2.5e5) + +# Testcase settings +set (HOMME_TEST_MOISTURE notdry) +set (HOMME_THETA_HY_MODE true) + +# Vert coord settings +set (HOMME_TEST_VCOORD_INT_FILE acme-72i.ascii) +set (HOMME_TEST_VCOORD_MID_FILE acme-72m.ascii) + +# Configure the namelist into the test directory +configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl + ${CMAKE_CURRENT_BINARY_DIR}/namelist.nl) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + scream/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + scream/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + scream/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + scream/mam4xx/physprops/ssam_rrtmg_c20240206.nc + scream/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/poly_rrtmg_c20240206.nc +) + +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS dynamics physics shoc cld p3 rrtmgp mam4_optics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml new file mode 100644 index 000000000000..1857bc534dd2 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/input.yaml @@ -0,0 +1,109 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + mass_column_conservation_error_tolerance: 1e-3 + energy_column_conservation_error_tolerance: 1e-4 + column_conservation_checks_fail_handling_type: Warning + property_check_data_fields: [phis] + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height : 1.0 + phis : 1.0 + + #variables required for shoc + surf_sens_flux: 0.0 + surf_evap: 0.0 + + #variables required for p3 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + + #variables required for mam4_aci + dgnum: 1e-3 + + #mam4 drydep + fv: 0.47569674433601039E+000 + landfrac: 0.14675684171817760E+000 + icefrac: 0.00000000000000000E+000 + ocnfrac: 0.85324315828182240E+000 + ram1: 0.45506166067091662E+002 + dgnumwet: 0.41417721820867320E-007 + wetdens: 0.15100083211582764E+004 + +atmosphere_processes: + atm_procs_list: [homme,physics] + schedule_type: Sequential + homme: + Moisture: moist + physics: + atm_procs_list: [mac_mic,mam4_optics,rrtmgp,mam4_drydep] + schedule_type: Sequential + Type: Group + mac_mic: + atm_procs_list: [shoc,CldFraction,mam4_aci,p3] + schedule_type: Sequential + Type: Group + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + mam4_aci: + wsubmin: 0.001 + top_level_mam4xx: 6 + p3: + do_prescribed_ccn: false + enable_column_conservation_checks: true + max_total_ni: 740.0e3 + shoc: + enable_column_conservation_checks: true + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + + mam4_optics: + mam4_mode1_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + mam4_mode2_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + mam4_mode3_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + mam4_mode4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + mam4_water_refindex_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + mam4_soa_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + mam4_dust_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + mam4_nacl_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ssam_rrtmg_c20240206.nc + mam4_so4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + mam4_pom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + mam4_bc_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + mam4_mom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/poly_rrtmg_c20240206.nc + + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + enable_column_conservation_checks: true + +grids_manager: + Type: Homme + physics_grid_type: GLL + dynamics_namelist_file_name: namelist.nl + vertical_coordinate_filename: IC_FILE + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml new file mode 100644 index 000000000000..5922c718e6e6 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep/output.yaml @@ -0,0 +1,123 @@ +%YAML 1.1 +--- +filename_prefix: homme_shoc_cld_mam_aci_p3_mam_optics_rrtmgp_mam_drydep +Averaging Type: Instant +Max Snapshots Per File: 1 +Fields: + Physics GLL: + Field Names: + # HOMME + - ps + - pseudo_density + - omega + - p_int + - p_mid + - pseudo_density_dry + - p_dry_int + - p_dry_mid + # SHOC + - cldfrac_liq + - eddy_diff_mom + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + HOMME + - horiz_winds + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + HOMME + - T_mid + #mam_optics + - aero_g_sw + - aero_ssa_sw + - aero_tau_lw + - aero_tau_sw + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net + # Diagnostics + - T_mid_at_lev_2 + - T_mid_at_model_top + - T_mid_at_model_bot + - T_mid_at_500mb + - T_mid_at_500hPa + - T_mid_at_50000Pa + - PotentialTemperature + - AtmosphereDensity + - Exner + - VirtualTemperature + - z_int + - geopotential_int_at_lev_2 + - z_mid_at_500mb + - geopotential_mid + - dz + - DryStaticEnergy + - SeaLevelPressure + - LiqWaterPath + - IceWaterPath + - VapWaterPath + - RainWaterPath + - RimeWaterPath + - ShortwaveCloudForcing + - LongwaveCloudForcing + - RelativeHumidity + - ZonalVapFlux + - MeridionalVapFlux + - PotentialTemperature_at_model_top + - PotentialTemperature_at_500mb + #drydep + - deposition_flux_of_cloud_borne_aerosols + - deposition_flux_of_interstitial_aerosols + # GLL output for homme states. These + # represent all current possible homme + # states available. + Dynamics: + Field Names: + - v_dyn + - vtheta_dp_dyn + - dp3d_dyn + - phi_int_dyn + - ps_dyn + - phis_dyn + - omega_dyn + - Qdp_dyn + IO Grid Name: Physics GLL + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml index 559f5158bb60..be4bb16b95d6 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml @@ -117,5 +117,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/CMakeLists.txt new file mode 100644 index 000000000000..41a915b92656 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/CMakeLists.txt @@ -0,0 +1,95 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Get or create the dynamics lib +# HOMME_TARGET NP PLEV QSIZE_D +CreateDynamicsLib("theta-l_kokkos" 4 72 41) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS cld_fraction shoc spa p3 scream_rrtmgp mam ${dynLibName} + LABELS dynamics shoc cld p3 rrtmgp physics mam4_wetscav + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Set homme's test options, so that we can configure the namelist correctly +# Discretization/algorithm settings +set (HOMME_TEST_NE 2) +set (HOMME_TEST_LIM 9) +set (HOMME_TEST_REMAP_FACTOR 3) +set (HOMME_TEST_TRACERS_FACTOR 1) +set (HOMME_TEST_TIME_STEP 300) +set (HOMME_THETA_FORM 1) +set (HOMME_TTYPE 5) +set (HOMME_SE_FTYPE 0) +set (HOMME_TEST_TRANSPORT_ALG 0) +set (HOMME_TEST_CUBED_SPHERE_MAP 0) + +# Hyperviscosity settings +set (HOMME_TEST_HVSCALING 0) +set (HOMME_TEST_HVS 1) +set (HOMME_TEST_HVS_TOM 0) +set (HOMME_TEST_HVS_Q 1) + +set (HOMME_TEST_NU 7e15) +set (HOMME_TEST_NUDIV 1e15) +set (HOMME_TEST_NUTOP 2.5e5) + +# Testcase settings +set (HOMME_TEST_MOISTURE notdry) +set (HOMME_THETA_HY_MODE true) + +# Vert coord settings +set (HOMME_TEST_VCOORD_INT_FILE acme-72i.ascii) +set (HOMME_TEST_VCOORD_MID_FILE acme-72m.ascii) + +# Configure the namelist into the test directory +configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl + ${CMAKE_CURRENT_BINARY_DIR}/namelist.nl) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/init/spa_init_ne2np4.nc + scream/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc + scream/init/${EAMxx_tests_IC_FILE_72lev} + cam/topo/${EAMxx_tests_TOPO_FILE} +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS dynamics physics shoc cld p3 rrtmgp spa mam4_wetscav + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml new file mode 100644 index 000000000000..c9b493a2958c --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/input.yaml @@ -0,0 +1,84 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 25.0 + phis : 0.1 + surf_evap: 0.0 + surf_sens_flux: 0.0 + precip_liq_surf_mass: 0.0 + precip_ice_surf_mass: 0.0 + #variables needed by mam4_wetscav + #--surface fluxes + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + +atmosphere_processes: + atm_procs_list: [homme,physics] + schedule_type: Sequential + homme: + Moisture: moist + physics: + atm_procs_list: [mac_aero_mic,rrtmgp] + schedule_type: Sequential + Type: Group + mac_aero_mic: + atm_procs_list: [shoc,CldFraction,spa,p3,mam4_wetscav] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + spa: + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc + p3: + max_total_ni: 740.0e3 + shoc: + check_flux_state_consistency: true + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + mam_wetscav: + number_of_subcycles: ${NUM_SUBCYCLES} + compute_tendencies: [all] + lambda_low: 0.001 + +grids_manager: + Type: Homme + physics_grid_type: GLL + dynamics_namelist_file_name: namelist.nl + vertical_coordinate_filename: IC_FILE + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml new file mode 100644 index 000000000000..afe8437a2cf4 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav/output.yaml @@ -0,0 +1,139 @@ +%YAML 1.1 +--- +filename_prefix: homme_shoc_cld_spa_p3_rrtmgp_mam4_wetscav_output +Averaging Type: Instant +Max Snapshots Per File: 1 +Fields: + Physics GLL: + Field Names: + # HOMME + - ps + - pseudo_density + - omega + - p_int + - p_mid + - pseudo_density_dry + - p_dry_int + - p_dry_mid + # SHOC + - cldfrac_liq + - eddy_diff_mom + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # SPA + - aero_g_sw + - aero_ssa_sw + - aero_tau_lw + - aero_tau_sw + - nccn + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + HOMME + - horiz_winds + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + HOMME + - T_mid + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net + # Diagnostics + - T_mid_at_lev_2 + - T_mid_at_model_top + - T_mid_at_model_bot + - T_mid_at_500mb + - T_mid_at_500hPa + - T_mid_at_50000Pa + - PotentialTemperature + - AtmosphereDensity + - Exner + - VirtualTemperature + - z_int + - geopotential_int_at_lev_2 + - z_mid_at_500mb + - geopotential_mid + - dz + - DryStaticEnergy + - SeaLevelPressure + - LiqWaterPath + - IceWaterPath + - VapWaterPath + - RainWaterPath + - RimeWaterPath + - ShortwaveCloudForcing + - LongwaveCloudForcing + - RelativeHumidity + - ZonalVapFlux + - MeridionalVapFlux + - PotentialTemperature_at_model_top + - PotentialTemperature_at_500mb + # MAM4_WETSCAV + - aerdepwetis + - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - pom_c1 + - pom_c3 + - pom_c4 + - so4_c1 + - so4_c2 + - so4_c3 + - soa_c1 + - soa_c2 + - soa_c3 + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... diff --git a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index b9c13234796b..bb23911ff1c2 100644 --- a/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -5,9 +5,13 @@ if (SCREAM_DOUBLE_PRECISION) if (SCREAM_ENABLE_MAM) add_subdirectory(mam/optics_rrtmgp) add_subdirectory(mam/shoc_mam4_aci) + add_subdirectory(mam/shoc_mam4_drydep) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_rrtmgp) add_subdirectory(mam/shoc_cldfrac_mam4_aci_p3_mam4_optics_rrtmgp) + add_subdirectory(mam/p3_mam4_wetscav) + add_subdirectory(mam/shoc_cldfrac_p3_wetscav) + add_subdirectory(mam/mam4_srf_online_emiss_mam4_constituent_fluxes) endif() endif() diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt new file mode 100644 index 000000000000..886936ca0846 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/CMakeLists.txt @@ -0,0 +1,53 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME mam4_srf_online_emiss_mam4_constituent_fluxes) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS mam + LABELS physics mam4_srf_online_emiss mam4_constituent_fluxes + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS physics mam4_srf_online_emiss mam4_constituent_fluxes + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml new file mode 100644 index 000000000000..55c62d69aa24 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -0,0 +1,47 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_srf_online_emiss, mam4_constituent_fluxes] + schedule_type: Sequential + mam4_srf_online_emiss: + # MAM4xx-Surface-Emissions + srf_remap_file: "" + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + pbl_height: 0.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml new file mode 100644 index 000000000000..c9cad5e2761a --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/output.yaml @@ -0,0 +1,41 @@ +%YAML 1.1 +--- +filename_prefix: mam4_srf_online_emiss_mam4_constituent_fluxes_output +Averaging Type: Instant +Field Names: + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - constituent_fluxes + - O3 + - H2O2 + - H2SO4 + - SO2 + - DMS + - SOAG +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml index 24e1dfbb3e7a..bdda48ebe459 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml @@ -26,5 +26,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/CMakeLists.txt new file mode 100644 index 000000000000..7e13932e0632 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME p3_mam4_wetscav) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS p3 mam + LABELS p3 physics mam4_wetscav + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep p3 dt<=300s +set (P3_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${P3_MAX_DT} - 1) / ${P3_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS p3 physics mam4_wetscav + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml new file mode 100644 index 000000000000..f0d9b7c89a3b --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/input.yaml @@ -0,0 +1,57 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic, mam4_wetscav] + schedule_type: Sequential + mac_mic: + atm_procs_list: [p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 25.0 + phis : 0.1 + #variables needed by mam4_wetscav + #--surface fluxes + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + + #variable required for p3 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml new file mode 100644 index 000000000000..c26db8eb6814 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/p3_mam4_wetscav/output.yaml @@ -0,0 +1,66 @@ +%YAML 1.1 +--- +filename_prefix: p3_mam4_wetscav_output +Averaging Type: Instant +Field Names: + - cldfrac_liq + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - aerdepwetis + - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/CMakeLists.txt new file mode 100644 index 000000000000..b5a6be819ce2 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_cldfrac_p3_wetscav) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc cld_fraction p3 mam + LABELS shoc cld p3 physics mam4_wetscav + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc cld p3 physics PEM mam4_wetscav + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml new file mode 100644 index 000000000000..804939a22058 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/input.yaml @@ -0,0 +1,73 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic, mam4_wetscav] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc,CldFraction,p3] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + max_total_ni: 740.0e3 + do_prescribed_ccn: false + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 25.0 + phis : 0.1 + #variables needed by mam4_wetscav + #--surface fluxes + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] + + #variable required for shoc + surf_sens_flux: 0.0 + surf_evap: 0.0 + + #variable required for p3 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml new file mode 100644 index 000000000000..1ae53e6c1c30 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_cldfrac_p3_wetscav/output.yaml @@ -0,0 +1,100 @@ +%YAML 1.1 +--- +filename_prefix: shoc_cldfrac_p3_wetscav_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + P3 + - qc + - qv + # wetscav + - T_mid + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - aerdepwetis + - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt new file mode 100644 index 000000000000..9f6fb8733c2c --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/CMakeLists.txt @@ -0,0 +1,41 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME shoc_mam4_drydep) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc mam + LABELS shoc physics mam4_drydep + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS shoc physics mam4_drydep + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml new file mode 100644 index 000000000000..c451008adc76 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/input.yaml @@ -0,0 +1,67 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mac_mic,mam4_drydep] + schedule_type: Sequential + mac_mic: + atm_procs_list: [shoc] + Type: Group + schedule_type: Sequential + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + + #variables needed for SHOC + surf_sens_flux: 0.0 + surf_evap: 0.0 + + #variables needed for mam4-drydep + fraction_landuse: 0.2 + w_updraft: 1e-5 + pbl_height: 0.0 + fv: 0.47569674433601039E+000 + landfrac: 0.14675684171817760E+000 + icefrac: 0.00000000000000000E+000 + ocnfrac: 0.85324315828182240E+000 + ram1: 0.45506166067091662E+002 + dgnumwet: 0.41417721820867320E-007 + wetdens: 0.15100083211582764E+004 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/output.yaml new file mode 100644 index 000000000000..d8c1291bc3c9 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/shoc_mam4_drydep/output.yaml @@ -0,0 +1,69 @@ +%YAML 1.1 +--- +filename_prefix: shoc_mam4_drydep_output +Averaging Type: Instant +Field Names: + # SHOC + - cldfrac_liq + - eddy_diff_mom + - horiz_winds + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - deposition_flux_of_cloud_borne_aerosols + - deposition_flux_of_interstitial_aerosols +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps +... diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt index 973e58b7da53..62c4b6d2266e 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt @@ -19,7 +19,6 @@ CreateUnitTest(create_vert_remap_and_weights "create_vert_remap_and_weights.cpp" # Run a test to setup nudging source data: set (NUM_STEPS 5) set (POSTFIX source_data) -set (ADD_RANKS false) set (ATM_TIME_STEP 300) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_source_data.yaml ${CMAKE_CURRENT_BINARY_DIR}/input_source_data.yaml) @@ -36,7 +35,6 @@ CreateUnitTestFromExec (shoc_p3_source shoc_p3_nudging set (NUM_STEPS 5) set (ATM_TIME_STEP 300) set (POSTFIX nudged) -set (ADD_RANKS true) set (VERT_TYPE TIME_DEPENDENT_3D_PROFILE) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging.yaml ${CMAKE_CURRENT_BINARY_DIR}/input_nudging.yaml) diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml index 2b33d15f9f75..6a4c0003376e 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml @@ -14,7 +14,7 @@ atmosphere_processes: p3: max_total_ni: 720.0e3 nudging: - nudging_filenames_patterns: [shoc_p3_source_data_${POSTFIX}.INSTANT.nsteps_x${NUM_STEPS}.${RUN_T0}.nc] + nudging_filenames_patterns: [shoc_p3_source_data_${POSTFIX}.INSTANT.nsteps_x${NUM_STEPS}.np1.${RUN_T0}.nc] nudging_fields: ["T_mid", "qv"] nudging_timescale: 1000 use_nudging_weights: true diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml index e740c5cdf1f1..d9b573068ae7 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml @@ -3,7 +3,6 @@ filename_prefix: shoc_p3_${POSTFIX}_nudged Averaging Type: Instant Max Snapshots Per File: 2 -MPI Ranks in Filename: ${ADD_RANKS} Field Names: - p_mid - T_mid diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml index c75a6a862f71..26f53fbe57f5 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml @@ -4,7 +4,6 @@ filename_prefix: shoc_p3_${POSTFIX}_nudged_remapped Averaging Type: Instant Max Snapshots Per File: 2 vertical_remap_file: vertical_remap.nc -MPI Ranks in Filename: ${ADD_RANKS} Field Names: - T_mid - qv diff --git a/components/eamxx/tests/python/pyp3/CMakeLists.txt b/components/eamxx/tests/python/pyp3/CMakeLists.txt index 7142be8dd8f8..945f15faaf65 100644 --- a/components/eamxx/tests/python/pyp3/CMakeLists.txt +++ b/components/eamxx/tests/python/pyp3/CMakeLists.txt @@ -53,6 +53,6 @@ foreach (rank IN LISTS MpiRanks) SRC_FILE p3_standalone_cxx.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc TGT_FILE p3_standalone_py.INSTANT.nsteps_x1.np${rank}.2021-10-12-45000.nc FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_cxx${suffix} ${FIXTURES_BASE_NAME}_py${suffix} - LABELS pyeamxx + LABELS pyscream ) endforeach() diff --git a/components/eamxx/tests/python/pyp3/p3_standalone_py b/components/eamxx/tests/python/pyp3/p3_standalone_py index 9d2b648060bf..34747943d081 100755 --- a/components/eamxx/tests/python/pyp3/p3_standalone_py +++ b/components/eamxx/tests/python/pyp3/p3_standalone_py @@ -5,8 +5,8 @@ import sys # Add path to scream libs sys.path.append('@SCREAM_BASE_DIR@/scripts') -# Add path to pyeamxx libs -sys.path.append('@CMAKE_BINARY_DIR@/src/python/libpyeamxx') +# Add path to pyscream libs +sys.path.append('@CMAKE_BINARY_DIR@/src/python') # Without these, and manual init/finalize, on my laptop I get # obscure MPI errors at exit. @@ -15,8 +15,7 @@ mpi4py.rc.initialize = False # do not initialize MPI automatically mpi4py.rc.finalize = False # do not finalize MPI automatically from mpi4py import MPI -import pyscream as ps -import pyp3 +import pyscream from pathlib import Path from utils import ensure_yaml @@ -34,17 +33,25 @@ def main (): dt = yaml_input['time_stepping']['time_step'] t0_str = yaml_input['time_stepping']['run_t0'] + ic_file = Path('@SCREAM_DATA_DIR@/init/screami_unit_tests_ne2np4L72_20220822.nc') + # Create the grid ncols = 218 nlevs = 72 - grid = ps.Grid("Physics",ncols,nlevs) + pyscream.create_grids_manager(ncols,nlevs,str(ic_file)) - ic_file = Path('@SCREAM_DATA_DIR@/init/screami_unit_tests_ne2np4L72_20220822.nc') + p3 = pyscream.AtmProc(yaml_input['atmosphere_processes']['p3'],'p3') + params = p3.get_params() + old = params.get_dbl('max_total_ni') + print (f"max_total_ni: {params.get_dbl('max_total_ni')}") + params.set("max_total_ni",1000000.0) + print (f"max_total_ni: {params.get_dbl('max_total_ni')}") + params.set("max_total_ni",old) + print (f"max_total_ni: {params.get_dbl('max_total_ni')}") - p3params = ps.ParameterList(yaml_input['atmosphere_processes']['p3']) - p3 = pyp3.P3(grid,p3params) missing = p3.read_ic(str(ic_file)) - print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") + if len(missing)>0: + print (f"WARNING! The following input fields were not found in the IC file, and must be manually initialized: {missing}") p3.initialize(t0_str) p3.setup_output("output_py.yaml") @@ -57,7 +64,7 @@ if __name__ == "__main__": # This level of indirection ensures all pybind structs are destroyed # before we finalize eamxx (and hence kokkos) MPI.Init() - ps.init() + pyscream.init() main () - ps.finalize() + pyscream.finalize() MPI.Finalize() diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt index c6b4e0748af3..40565d94bf8e 100644 --- a/components/eamxx/tests/single-process/CMakeLists.txt +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -21,6 +21,10 @@ if (SCREAM_ENABLE_MAM) # initial conditions. add_subdirectory(mam/optics) add_subdirectory(mam/aci) + add_subdirectory(mam/drydep) + add_subdirectory(mam/wet_scav) + add_subdirectory(mam/emissions) + add_subdirectory(mam/constituent_fluxes) endif() if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) add_subdirectory(zm) diff --git a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt index 0d5d60ee7d5a..490dc389d952 100644 --- a/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aci/CMakeLists.txt @@ -24,6 +24,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml # Ensure test input files are present in the data dir GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) # Compare output files produced by npX tests, to ensure they are bfb include (CompareNCFiles) diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt b/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt new file mode 100644 index 000000000000..2615f152b264 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/CMakeLists.txt @@ -0,0 +1,44 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_constituent_fluxes_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_constituent_fluxes physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_constituent_fluxes physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml b/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml new file mode 100644 index 000000000000..d012274d1469 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/input.yaml @@ -0,0 +1,37 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_constituent_fluxes] +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + phis : 1.0 + #These should come from the input file + + #we should get the following variables from other processes + pbl_height : 1.0 + constituent_fluxes: 1.e-5 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml new file mode 100644 index 000000000000..b237056b5cc1 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/constituent_fluxes/output.yaml @@ -0,0 +1,43 @@ +%YAML 1.1 +--- +filename_prefix: mam4_constituent_fluxes_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - O3 + - H2O2 + - H2SO4 + - SO2 + - DMS + - SOAG + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + +output_control: + Frequency: 1 + frequency_units: nsteps +... diff --git a/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt new file mode 100644 index 000000000000..d05e6823642f --- /dev/null +++ b/components/eamxx/tests/single-process/mam/drydep/CMakeLists.txt @@ -0,0 +1,46 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_drydep_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_drydep physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_drydep physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() + diff --git a/components/eamxx/tests/single-process/mam/drydep/input.yaml b/components/eamxx/tests/single-process/mam/drydep/input.yaml new file mode 100644 index 000000000000..86c6dd8766a2 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/drydep/input.yaml @@ -0,0 +1,44 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_drydep] + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + fraction_landuse: 0.2 + w_updraft: 1e-5 + pbl_height: 0.0 + obklen: -0.48167567011091336E+002 + fv: 0.47569674433601039E+000 + landfrac: 0.14675684171817760E+000 + icefrac: 0.00000000000000000E+000 + ocnfrac: 0.85324315828182240E+000 + ustar: 0.27482164092221828E+000 + ram1: 0.45506166067091662E+002 + dgnumwet: 0.41417721820867320E-007 + wetdens: 0.15100083211582764E+004 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/drydep/output.yaml b/components/eamxx/tests/single-process/mam/drydep/output.yaml new file mode 100644 index 000000000000..6d8a79e8fe00 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/drydep/output.yaml @@ -0,0 +1,64 @@ +%YAML 1.1 +--- +filename_prefix: mam4_drydep_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - deposition_flux_of_cloud_borne_aerosols + - deposition_flux_of_interstitial_aerosols + +output_control: + Frequency: 1 + frequency_units: nsteps +... diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt new file mode 100644 index 000000000000..c8e690b929fc --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -0,0 +1,60 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_srf_online_emiss_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_srf_online_emiss physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 2.5h 24h +set (RUN_T0 2021-10-12-45000) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc +) +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_srf_online_emiss physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml new file mode 100644 index 000000000000..e7af45178aa5 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -0,0 +1,49 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_srf_online_emiss] + mam4_srf_online_emiss: + # MAM4xx-Surface-Emissions + srf_remap_file: "" + srf_emis_specifier_for_DMS: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/DMSflux.2010.ne2np4_conserv.POPmonthlyClimFromACES4BGC_c20240726.nc + srf_emis_specifier_for_SO2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so2_surf_ne2np4_2010_clim_c20240723.nc + srf_emis_specifier_for_bc_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_bc_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a2_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_num_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_num_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc + srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + phis : 1.0 + #These should come from the input file + + #we should get the following variables from other processes + pbl_height : 1.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/emissions/output.yaml b/components/eamxx/tests/single-process/mam/emissions/output.yaml new file mode 100644 index 000000000000..433f7d58fa40 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/emissions/output.yaml @@ -0,0 +1,12 @@ +%YAML 1.1 +--- +filename_prefix: mam4_srf_online_emiss_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - constituent_fluxes +output_control: + Frequency: 1 + frequency_units: nsteps +... diff --git a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt index a3e6f64ec7d0..0a3a0b877c5e 100644 --- a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt @@ -40,7 +40,6 @@ set (TEST_INPUT_FILES scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc scream/mam4xx/physprops/poly_rrtmg_c20240206.nc - scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) @@ -63,4 +62,4 @@ if (SCREAM_ENABLE_BASELINE_TESTS) # Note: one is enough, since we already check that np1 is BFB with npX set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) -endif() \ No newline at end of file +endif() diff --git a/components/eamxx/tests/single-process/mam/optics/output.yaml b/components/eamxx/tests/single-process/mam/optics/output.yaml index 8ea3d237933d..d0c27e042843 100644 --- a/components/eamxx/tests/single-process/mam/optics/output.yaml +++ b/components/eamxx/tests/single-process/mam/optics/output.yaml @@ -13,5 +13,4 @@ Fields: output_control: Frequency: 2 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt new file mode 100644 index 000000000000..3881df3e059d --- /dev/null +++ b/components/eamxx/tests/single-process/mam/wet_scav/CMakeLists.txt @@ -0,0 +1,43 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_wetscav_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_wetscav physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) +set (ATM_TIME_STEP 1800) +set (RUN_T0 2021-10-12-45000) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_wetscav physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/single-process/mam/wet_scav/input.yaml b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml new file mode 100644 index 000000000000..d1fd9f3659da --- /dev/null +++ b/components/eamxx/tests/single-process/mam/wet_scav/input.yaml @@ -0,0 +1,45 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_wetscav] + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height: 25.0 + phis : 0.1 + #variables from P3 + precip_total_tend: 1e-5 + nevapr: 1e-5 + #surface fluxes + wetdep_hydrophilic_bc: 1e-5 # wet deposition of hydrophilic black carbon [kg/m2/s] + drydep_hydrophilic_bc: 1e-5 # dry deposition of hydrophilic black carbon [kg/m2/s] + wetdep_hydrophilic_oc: 1e-5 # wet deposition of hydrophilic organic carbon [kg/m2/s] + drydep_hydrophilic_oc: 1e-5 # dry deposition of hydrophilic organic carbon [kg/m2/s] + wetdep_dust_bin1: 1e-5 # wet deposition of dust (bin1) [kg/m2/s] + wetdep_dust_bin2: 1e-5 # wet deposition of dust (bin2) [kg/m2/s] + wetdep_dust_bin3: 1e-5 # wet deposition of dust (bin3) [kg/m2/s] + wetdep_dust_bin4: 1e-5 # wet deposition of dust (bin4) [kg/m2/s] +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/wet_scav/output.yaml b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml new file mode 100644 index 000000000000..9cd0a16bc414 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/wet_scav/output.yaml @@ -0,0 +1,42 @@ +%YAML 1.1 +--- +filename_prefix: mam4_wetscav_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - aerdepwetis + - aerdepwetcw + - dgnumwet + - dgncur_a + - wetdens + - qaerwat + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - num_c1 + - num_c2 + - num_c3 + - num_c4 + - pom_c1 + - pom_c3 + - pom_c4 + - so4_c1 + - so4_c2 + - so4_c3 + - soa_c1 + - soa_c2 + - soa_c3 +output_control: + Frequency: 1 + frequency_units: nsteps +... diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index 9e6e94a2164f..dc1ece59b287 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -422,6 +422,18 @@ TEST_CASE("rrtmgp_scream_standalone", "") { #endif #ifdef RRTMGP_ENABLE_KOKKOS TEST_CASE("rrtmgp_scream_standalone_k", "") { +#ifdef RRTMGP_LAYOUT_LEFT + using layout_t = Kokkos::LayoutLeft; +#else + using layout_t = typename ekat::KokkosTypes::Layout; +#endif + using interface_t = scream::rrtmgp::rrtmgp_interface; + using MDRP = interface_t::MDRP; + using utils_t = rrtmgpTest::rrtmgp_test_utils; + using real1dk = interface_t::view_t; + using real2dk = interface_t::view_t; + using real3dk = interface_t::view_t; + // Get baseline name (needs to be passed as an arg) std::string inputfile = ekat::TestSession::get().params.at("rrtmgp_inputfile"); std::string baseline = ekat::TestSession::get().params.at("rrtmgp_baseline"); @@ -439,7 +451,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { real2dk sw_flux_dn_dir_ref; real2dk lw_flux_up_ref; real2dk lw_flux_dn_ref; - rrtmgpTest::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); + utils_t::read_fluxes(baseline, sw_flux_up_ref, sw_flux_dn_ref, sw_flux_dn_dir_ref, lw_flux_up_ref, lw_flux_dn_ref ); // Load ad parameter list std::string fname = "input_unit.yaml"; @@ -526,11 +538,11 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { // this will copy the first column of the input data (the first profile) ncol // times. We will then fill some fraction of these columns with clouds for // the test problem. - GasConcsK gas_concs; + GasConcsK gas_concs; real2dk col_dry; read_atmos(inputfile, p_lay_all, t_lay_all, p_lev_all, t_lev_all, gas_concs, col_dry, ncol_all); // Setup dummy problem; need to use tmp arrays with ncol_all size - rrtmgpTest::dummy_atmos( + utils_t::dummy_atmos( inputfile, ncol_all, p_lay_all, t_lay_all, sfc_alb_dir_vis_all, sfc_alb_dir_nir_all, sfc_alb_dif_vis_all, sfc_alb_dif_nir_all, @@ -547,7 +559,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { sfc_alb_dif_nir(icol) = sfc_alb_dif_nir_all(icol_all); mu0(icol) = mu0_all(icol_all); }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; p_lay(icol,ilay) = p_lay_all(icol_all,ilay); t_lay(icol,ilay) = t_lay_all(icol_all,ilay); @@ -557,14 +569,14 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { rei(icol,ilay) = rei_all(icol_all,ilay); cld(icol,ilay) = cld_all(icol_all,ilay); }); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; p_lev(icol,ilay) = p_lev_all(icol_all,ilay); t_lev(icol,ilay) = t_lev_all(icol_all,ilay); }); // Need to calculate a dummy pseudo_density for our test problem - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { p_del(icol,ilay) = abs(p_lev(icol,ilay+1) - p_lev(icol,ilay)); }); @@ -573,13 +585,13 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { auto qc = real2dk("qc", ncol, nlay); auto nc = real2dk("nc", ncol, nlay); auto qi = real2dk("qi", ncol, nlay); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { qc(icol,ilay) = 1e-3 * lwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); qi(icol,ilay) = 1e-3 * iwp(icol,ilay) * cld(icol,ilay) * PC::gravit / p_del(icol,ilay); }); // Copy gases from gas_concs to gas_vmr array - Kokkos::parallel_for(conv::get_mdrp<3>({ncol,nlay,ngas}), KOKKOS_LAMBDA(int icol, int ilay, int igas) { + Kokkos::parallel_for(MDRP::template get<3>({ncol,nlay,ngas}), KOKKOS_LAMBDA(int icol, int ilay, int igas) { auto icol_all = ncol * irank + icol; gas_vmr(icol,ilay,igas) = gas_concs.concs(icol_all,ilay,igas); }); @@ -691,7 +703,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { const int i = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay+1), [&] (const int& k) { - if (k < nlay) t_lay(i+1,k+1) = d_tmid(i,k); + if (k < nlay) t_lay(i,k) = d_tmid(i,k); sw_flux_up_test(i,k) = d_sw_flux_up(i,k); sw_flux_dn_test(i,k) = d_sw_flux_dn(i,k); @@ -722,7 +734,7 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { auto sw_flux_dn_dir_loc = real2dk("sw_flux_dn_dir_loc", ncol, nlay+1); auto lw_flux_up_loc = real2dk("lw_flux_up_loc" , ncol, nlay+1); auto lw_flux_dn_loc = real2dk("lw_flux_dn_loc" , ncol, nlay+1); - Kokkos::parallel_for(conv::get_mdrp<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { + Kokkos::parallel_for(MDRP::template get<2>({nlay+1,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { auto icol_all = ncol * irank + icol; sw_flux_up_loc(icol,ilay) = sw_flux_up_ref(icol_all,ilay); sw_flux_dn_loc(icol,ilay) = sw_flux_dn_ref(icol_all,ilay); @@ -730,11 +742,11 @@ TEST_CASE("rrtmgp_scream_standalone_k", "") { lw_flux_up_loc(icol,ilay) = lw_flux_up_ref(icol_all,ilay); lw_flux_dn_loc(icol,ilay) = lw_flux_dn_ref(icol_all,ilay); }); - REQUIRE(rrtmgpTest::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); - REQUIRE(rrtmgpTest::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_up_loc , sw_flux_up_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_dn_loc , sw_flux_dn_test , 1.0)); + REQUIRE(utils_t::all_close(sw_flux_dn_dir_loc, sw_flux_dn_dir_test, 1.0)); + REQUIRE(utils_t::all_close(lw_flux_up_loc , lw_flux_up_test , 1.0)); + REQUIRE(utils_t::all_close(lw_flux_dn_loc , lw_flux_dn_test , 1.0)); // Finalize the driver. YAKL will be finalized inside // RRTMGPRadiation::finalize_impl after RRTMGP has had the diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index cf3c7ef0b091..a373ae84e47b 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -313,6 +313,8 @@ lnd/clm2/surfdata_map/surfdata_ne120pg2_simyr2010_c230119.nc lnd/clm2/surfdata_map/surfdata_ne256pg2_simyr1850_c240131.nc lnd/clm2/surfdata_map/surfdata_ne256pg2_simyr2010_c230207.nc + +lnd/clm2/surfdata_map/surfdata_ne512pg2_simyr2010_c240522.nc lnd/clm2/surfdata_map/surfdata_ne1024pg2_simyr2010_c211021.nc diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 8afa64f4f5a1..7965c19be34f 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -1483,7 +1483,7 @@ CLM run type. +"512x1024,360x720cru,128x256,64x128,48x96,32x64,8x16,94x192,0.23x0.31,0.9x1.25,1.9x2.5,2.5x3.33,4x5,10x15,5x5_amazon,1x1_tropicAtl,1x1_camdenNJ,1x1_vancouverCAN,1x1_mexicocityMEX,1x1_asphaltjungleNJ,1x1_brazil,1x1_urbanc_alpha,1x1_numaIA,1x1_smallvilleIA,0.1x0.1,0.5x0.5,3x3min,5x5min,10x10min,0.33x0.33,ne4np4,ne4np4.pg2,ne11np4,ne16np4,ne30np4,ne30np4.pg2,ne60np4,ne120np4,ne120np4.pg2,ne240np4,ne256np4,ne256np4.pg2,ne512np4.pg2,ne1024np4,ne1024np4.pg2,1km-merge-10min,ne0np4_arm_x8v3_lowcon,ne0np4_conus_x4v1_lowcon,ne0np4_enax4v1,ne0np4_twpx4v1,r2,r05,r0125,NLDAS,ne0np4_northamericax4v1.pg2,ne0np4_arcticx4v1.pg2,r025"> Horizontal resolutions Note: 0.1x0.1, 0.5x0.5, 5x5min, 10x10min, 3x3min and 0.33x0.33 are only used for CLM tools diff --git a/components/homme/src/share/semoab_mod.F90 b/components/homme/src/share/semoab_mod.F90 index 2d8bb099ade3..4841cb84b543 100644 --- a/components/homme/src/share/semoab_mod.F90 +++ b/components/homme/src/share/semoab_mod.F90 @@ -23,7 +23,7 @@ module semoab_mod use seq_comm_mct, only: MHPGID ! app id on moab side, for PGx style mesh, uniform from se use seq_comm_mct, only: atm_pg_active ! turn it on when PG style mesh active - use dyn_grid, only: fv_nphys ! phys grid mesh will be replicated too + !use dyn_grid, only: fv_nphys ! phys grid mesh will be replicated too use gllfvremap_mod, only: gfr_f_get_corner_latlon use control_mod, only : west, east, south, north ! 1, 2, 3, 4 @@ -76,7 +76,7 @@ integer function search_in(intarr, leng, value) end function search_in - subroutine create_moab_meshes(par, elem) + subroutine create_moab_meshes(par, elem, fv_nphys) use ISO_C_BINDING use iMOAB, only: iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_CreateElements, & @@ -86,6 +86,7 @@ subroutine create_moab_meshes(par, elem) type (element_t), intent(inout) :: elem(:) type (parallel_t) , intent(in) :: par + integer , intent(in) :: fv_nphys integer ierr, i, j, ie, iv, block_ID, k, numvals integer icol, irow, je, linx ! local indices in fine el connect diff --git a/components/homme/src/theta-l_kokkos/CMakeLists.txt b/components/homme/src/theta-l_kokkos/CMakeLists.txt index 739866960e36..a585090ea76a 100644 --- a/components/homme/src/theta-l_kokkos/CMakeLists.txt +++ b/components/homme/src/theta-l_kokkos/CMakeLists.txt @@ -106,6 +106,7 @@ MACRO(THETAL_KOKKOS_SETUP) ${SRC_SHARE_DIR}/planar_mod.F90 ${SRC_SHARE_DIR}/geometry_mod.F90 ${SRC_SHARE_DIR}/planar_mesh_mod.F90 + ${SRC_SHARE_DIR}/semoab_mod.F90 ) IF (NOT HOMME_ENABLE_COMPOSE) diff --git a/driver-mct/cime_config/namelist_definition_drv.xml b/driver-mct/cime_config/namelist_definition_drv.xml index 7fbf83688c8a..982acabe64d8 100644 --- a/driver-mct/cime_config/namelist_definition_drv.xml +++ b/driver-mct/cime_config/namelist_definition_drv.xml @@ -1527,6 +1527,7 @@ 1.e-10 1.e-10 1.e-10 + 1.e-10 diff --git a/driver-mct/main/cime_comp_mod.F90 b/driver-mct/main/cime_comp_mod.F90 index 2131d0c86844..7cf442b9413d 100644 --- a/driver-mct/main/cime_comp_mod.F90 +++ b/driver-mct/main/cime_comp_mod.F90 @@ -714,6 +714,9 @@ module cime_comp_mod subroutine cime_pre_init1(esmf_log_option) use shr_pio_mod, only : shr_pio_init1, shr_pio_init2 use seq_comm_mct, only: num_inst_driver +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) + use atm_comp_mct, only: atm_init_hip_mct +#endif !---------------------------------------------------------- !| Initialize MCT and MPI communicators and IO !---------------------------------------------------------- @@ -736,6 +739,9 @@ subroutine cime_pre_init1(esmf_log_option) beg_count = shr_sys_irtc(irtc_rate) +#if defined(SCREAM_SYSTEM_WORKAROUND) && (SCREAM_SYSTEM_WORKAROUND == 1) + call atm_init_hip_mct() +#endif call mpi_init(ierr) call shr_mpi_chkerr(ierr,subname//' mpi_init') diff --git a/externals/mam4xx b/externals/mam4xx index a8aa56ba915c..2eee1988b1e6 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit a8aa56ba915c83c22edc54ce5fd8ae4a23e6173e +Subproject commit 2eee1988b1e6488c904e0a28564bdcadfa190d34 diff --git a/share/streams/shr_dmodel_mod.F90 b/share/streams/shr_dmodel_mod.F90 index eea74a66cdee..e1a7c39d783e 100644 --- a/share/streams/shr_dmodel_mod.F90 +++ b/share/streams/shr_dmodel_mod.F90 @@ -383,7 +383,7 @@ subroutine shr_dmodel_readgrid( gGrid, gsMap, nxgo, nygo, nzgo, filename, compid nxgo = scm_nx nygo = scm_ny nzgo = -1 - gsize = abs(scm_nx*scm_nx*nzgo) + gsize = abs(scm_nx*scm_ny*nzgo) else nxgo = nxg nygo = nyg